+
+## License
+
+The [BSD 3-Clause license][bsd], the same as the [Go language][golic]. Cascadia's license is [here][caslic].
+
+[jquery]: http://jquery.com/
+[go]: http://golang.org/
+[cascadia]: https://github.com/andybalholm/cascadia
+[cascadiacli]: https://github.com/suntong/cascadia
+[bsd]: http://opensource.org/licenses/BSD-3-Clause
+[golic]: http://golang.org/LICENSE
+[caslic]: https://github.com/andybalholm/cascadia/blob/master/LICENSE
+[doc]: http://godoc.org/github.com/PuerkitoBio/goquery
+[index]: http://api.jquery.com/index/
+[gonet]: https://github.com/golang/net/
+[html]: http://godoc.org/golang.org/x/net/html
+[wiki]: https://github.com/PuerkitoBio/goquery/wiki/Tips-and-tricks
+[thatguystone]: https://github.com/thatguystone
+[piotr]: https://github.com/piotrkowalczuk
+[goq]: https://github.com/andrewstuart/goq
diff --git a/vendor/github.com/PuerkitoBio/goquery/array.go b/vendor/github.com/PuerkitoBio/goquery/array.go
new file mode 100644
index 0000000..1b1f6cb
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/array.go
@@ -0,0 +1,124 @@
+package goquery
+
+import (
+ "golang.org/x/net/html"
+)
+
+const (
+ maxUint = ^uint(0)
+ maxInt = int(maxUint >> 1)
+
+ // ToEnd is a special index value that can be used as end index in a call
+ // to Slice so that all elements are selected until the end of the Selection.
+ // It is equivalent to passing (*Selection).Length().
+ ToEnd = maxInt
+)
+
+// First reduces the set of matched elements to the first in the set.
+// It returns a new Selection object, and an empty Selection object if the
+// the selection is empty.
+func (s *Selection) First() *Selection {
+ return s.Eq(0)
+}
+
+// Last reduces the set of matched elements to the last in the set.
+// It returns a new Selection object, and an empty Selection object if
+// the selection is empty.
+func (s *Selection) Last() *Selection {
+ return s.Eq(-1)
+}
+
+// Eq reduces the set of matched elements to the one at the specified index.
+// If a negative index is given, it counts backwards starting at the end of the
+// set. It returns a new Selection object, and an empty Selection object if the
+// index is invalid.
+func (s *Selection) Eq(index int) *Selection {
+ if index < 0 {
+ index += len(s.Nodes)
+ }
+
+ if index >= len(s.Nodes) || index < 0 {
+ return newEmptySelection(s.document)
+ }
+
+ return s.Slice(index, index+1)
+}
+
+// Slice reduces the set of matched elements to a subset specified by a range
+// of indices. The start index is 0-based and indicates the index of the first
+// element to select. The end index is 0-based and indicates the index at which
+// the elements stop being selected (the end index is not selected).
+//
+// The indices may be negative, in which case they represent an offset from the
+// end of the selection.
+//
+// The special value ToEnd may be specified as end index, in which case all elements
+// until the end are selected. This works both for a positive and negative start
+// index.
+func (s *Selection) Slice(start, end int) *Selection {
+ if start < 0 {
+ start += len(s.Nodes)
+ }
+ if end == ToEnd {
+ end = len(s.Nodes)
+ } else if end < 0 {
+ end += len(s.Nodes)
+ }
+ return pushStack(s, s.Nodes[start:end])
+}
+
+// Get retrieves the underlying node at the specified index.
+// Get without parameter is not implemented, since the node array is available
+// on the Selection object.
+func (s *Selection) Get(index int) *html.Node {
+ if index < 0 {
+ index += len(s.Nodes) // Negative index gets from the end
+ }
+ return s.Nodes[index]
+}
+
+// Index returns the position of the first element within the Selection object
+// relative to its sibling elements.
+func (s *Selection) Index() int {
+ if len(s.Nodes) > 0 {
+ return newSingleSelection(s.Nodes[0], s.document).PrevAll().Length()
+ }
+ return -1
+}
+
+// IndexSelector returns the position of the first element within the
+// Selection object relative to the elements matched by the selector, or -1 if
+// not found.
+func (s *Selection) IndexSelector(selector string) int {
+ if len(s.Nodes) > 0 {
+ sel := s.document.Find(selector)
+ return indexInSlice(sel.Nodes, s.Nodes[0])
+ }
+ return -1
+}
+
+// IndexMatcher returns the position of the first element within the
+// Selection object relative to the elements matched by the matcher, or -1 if
+// not found.
+func (s *Selection) IndexMatcher(m Matcher) int {
+ if len(s.Nodes) > 0 {
+ sel := s.document.FindMatcher(m)
+ return indexInSlice(sel.Nodes, s.Nodes[0])
+ }
+ return -1
+}
+
+// IndexOfNode returns the position of the specified node within the Selection
+// object, or -1 if not found.
+func (s *Selection) IndexOfNode(node *html.Node) int {
+ return indexInSlice(s.Nodes, node)
+}
+
+// IndexOfSelection returns the position of the first node in the specified
+// Selection object within this Selection object, or -1 if not found.
+func (s *Selection) IndexOfSelection(sel *Selection) int {
+ if sel != nil && len(sel.Nodes) > 0 {
+ return indexInSlice(s.Nodes, sel.Nodes[0])
+ }
+ return -1
+}
diff --git a/vendor/github.com/PuerkitoBio/goquery/array_test.go b/vendor/github.com/PuerkitoBio/goquery/array_test.go
new file mode 100644
index 0000000..7857b38
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/array_test.go
@@ -0,0 +1,234 @@
+package goquery
+
+import (
+ "testing"
+)
+
+func TestFirst(t *testing.T) {
+ sel := Doc().Find(".pvk-content").First()
+ assertLength(t, sel.Nodes, 1)
+}
+
+func TestFirstEmpty(t *testing.T) {
+ sel := Doc().Find(".pvk-zzcontentzz").First()
+ assertLength(t, sel.Nodes, 0)
+}
+
+func TestFirstInvalid(t *testing.T) {
+ sel := Doc().Find("").First()
+ assertLength(t, sel.Nodes, 0)
+}
+
+func TestFirstRollback(t *testing.T) {
+ sel := Doc().Find(".pvk-content")
+ sel2 := sel.First().End()
+ assertEqual(t, sel, sel2)
+}
+
+func TestLast(t *testing.T) {
+ sel := Doc().Find(".pvk-content").Last()
+ assertLength(t, sel.Nodes, 1)
+
+ // Should contain Footer
+ foot := Doc().Find(".footer")
+ if !sel.Contains(foot.Nodes[0]) {
+ t.Error("Last .pvk-content should contain .footer.")
+ }
+}
+
+func TestLastEmpty(t *testing.T) {
+ sel := Doc().Find(".pvk-zzcontentzz").Last()
+ assertLength(t, sel.Nodes, 0)
+}
+
+func TestLastInvalid(t *testing.T) {
+ sel := Doc().Find("").Last()
+ assertLength(t, sel.Nodes, 0)
+}
+
+func TestLastRollback(t *testing.T) {
+ sel := Doc().Find(".pvk-content")
+ sel2 := sel.Last().End()
+ assertEqual(t, sel, sel2)
+}
+
+func TestEq(t *testing.T) {
+ sel := Doc().Find(".pvk-content").Eq(1)
+ assertLength(t, sel.Nodes, 1)
+}
+
+func TestEqNegative(t *testing.T) {
+ sel := Doc().Find(".pvk-content").Eq(-1)
+ assertLength(t, sel.Nodes, 1)
+
+ // Should contain Footer
+ foot := Doc().Find(".footer")
+ if !sel.Contains(foot.Nodes[0]) {
+ t.Error("Index -1 of .pvk-content should contain .footer.")
+ }
+}
+
+func TestEqEmpty(t *testing.T) {
+ sel := Doc().Find("something_random_that_does_not_exists").Eq(0)
+ assertLength(t, sel.Nodes, 0)
+}
+
+func TestEqInvalid(t *testing.T) {
+ sel := Doc().Find("").Eq(0)
+ assertLength(t, sel.Nodes, 0)
+}
+
+func TestEqInvalidPositive(t *testing.T) {
+ sel := Doc().Find(".pvk-content").Eq(3)
+ assertLength(t, sel.Nodes, 0)
+}
+
+func TestEqInvalidNegative(t *testing.T) {
+ sel := Doc().Find(".pvk-content").Eq(-4)
+ assertLength(t, sel.Nodes, 0)
+}
+
+func TestEqRollback(t *testing.T) {
+ sel := Doc().Find(".pvk-content")
+ sel2 := sel.Eq(1).End()
+ assertEqual(t, sel, sel2)
+}
+
+func TestSlice(t *testing.T) {
+ sel := Doc().Find(".pvk-content").Slice(0, 2)
+
+ assertLength(t, sel.Nodes, 2)
+ assertSelectionIs(t, sel, "#pc1", "#pc2")
+}
+
+func TestSliceToEnd(t *testing.T) {
+ sel := Doc().Find(".pvk-content").Slice(1, ToEnd)
+
+ assertLength(t, sel.Nodes, 2)
+ assertSelectionIs(t, sel.Eq(0), "#pc2")
+ if _, ok := sel.Eq(1).Attr("id"); ok {
+ t.Error("Want no attribute ID, got one")
+ }
+}
+
+func TestSliceEmpty(t *testing.T) {
+ defer assertPanic(t)
+ Doc().Find("x").Slice(0, 2)
+}
+
+func TestSliceInvalid(t *testing.T) {
+ defer assertPanic(t)
+ Doc().Find("").Slice(0, 2)
+}
+
+func TestSliceInvalidToEnd(t *testing.T) {
+ defer assertPanic(t)
+ Doc().Find("").Slice(2, ToEnd)
+}
+
+func TestSliceOutOfBounds(t *testing.T) {
+ defer assertPanic(t)
+ Doc().Find(".pvk-content").Slice(2, 12)
+}
+
+func TestNegativeSliceStart(t *testing.T) {
+ sel := Doc().Find(".container-fluid").Slice(-2, 3)
+ assertLength(t, sel.Nodes, 1)
+ assertSelectionIs(t, sel.Eq(0), "#cf3")
+}
+
+func TestNegativeSliceEnd(t *testing.T) {
+ sel := Doc().Find(".container-fluid").Slice(1, -1)
+ assertLength(t, sel.Nodes, 2)
+ assertSelectionIs(t, sel.Eq(0), "#cf2")
+ assertSelectionIs(t, sel.Eq(1), "#cf3")
+}
+
+func TestNegativeSliceBoth(t *testing.T) {
+ sel := Doc().Find(".container-fluid").Slice(-3, -1)
+ assertLength(t, sel.Nodes, 2)
+ assertSelectionIs(t, sel.Eq(0), "#cf2")
+ assertSelectionIs(t, sel.Eq(1), "#cf3")
+}
+
+func TestNegativeSliceToEnd(t *testing.T) {
+ sel := Doc().Find(".container-fluid").Slice(-3, ToEnd)
+ assertLength(t, sel.Nodes, 3)
+ assertSelectionIs(t, sel, "#cf2", "#cf3", "#cf4")
+}
+
+func TestNegativeSliceOutOfBounds(t *testing.T) {
+ defer assertPanic(t)
+ Doc().Find(".container-fluid").Slice(-12, -7)
+}
+
+func TestSliceRollback(t *testing.T) {
+ sel := Doc().Find(".pvk-content")
+ sel2 := sel.Slice(0, 2).End()
+ assertEqual(t, sel, sel2)
+}
+
+func TestGet(t *testing.T) {
+ sel := Doc().Find(".pvk-content")
+ node := sel.Get(1)
+ if sel.Nodes[1] != node {
+ t.Errorf("Expected node %v to be %v.", node, sel.Nodes[1])
+ }
+}
+
+func TestGetNegative(t *testing.T) {
+ sel := Doc().Find(".pvk-content")
+ node := sel.Get(-3)
+ if sel.Nodes[0] != node {
+ t.Errorf("Expected node %v to be %v.", node, sel.Nodes[0])
+ }
+}
+
+func TestGetInvalid(t *testing.T) {
+ defer assertPanic(t)
+ sel := Doc().Find(".pvk-content")
+ sel.Get(129)
+}
+
+func TestIndex(t *testing.T) {
+ sel := Doc().Find(".pvk-content")
+ if i := sel.Index(); i != 1 {
+ t.Errorf("Expected index of 1, got %v.", i)
+ }
+}
+
+func TestIndexSelector(t *testing.T) {
+ sel := Doc().Find(".hero-unit")
+ if i := sel.IndexSelector("div"); i != 4 {
+ t.Errorf("Expected index of 4, got %v.", i)
+ }
+}
+
+func TestIndexSelectorInvalid(t *testing.T) {
+ sel := Doc().Find(".hero-unit")
+ if i := sel.IndexSelector(""); i != -1 {
+ t.Errorf("Expected index of -1, got %v.", i)
+ }
+}
+
+func TestIndexOfNode(t *testing.T) {
+ sel := Doc().Find("div.pvk-gutter")
+ if i := sel.IndexOfNode(sel.Nodes[1]); i != 1 {
+ t.Errorf("Expected index of 1, got %v.", i)
+ }
+}
+
+func TestIndexOfNilNode(t *testing.T) {
+ sel := Doc().Find("div.pvk-gutter")
+ if i := sel.IndexOfNode(nil); i != -1 {
+ t.Errorf("Expected index of -1, got %v.", i)
+ }
+}
+
+func TestIndexOfSelection(t *testing.T) {
+ sel := Doc().Find("div")
+ sel2 := Doc().Find(".hero-unit")
+ if i := sel.IndexOfSelection(sel2); i != 4 {
+ t.Errorf("Expected index of 4, got %v.", i)
+ }
+}
diff --git a/vendor/github.com/PuerkitoBio/goquery/bench_array_test.go b/vendor/github.com/PuerkitoBio/goquery/bench_array_test.go
new file mode 100644
index 0000000..29c7e20
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/bench_array_test.go
@@ -0,0 +1,120 @@
+package goquery
+
+import (
+ "testing"
+)
+
+func BenchmarkFirst(b *testing.B) {
+ b.StopTimer()
+ sel := DocB().Find("dd")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ sel.First()
+ }
+}
+
+func BenchmarkLast(b *testing.B) {
+ b.StopTimer()
+ sel := DocB().Find("dd")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ sel.Last()
+ }
+}
+
+func BenchmarkEq(b *testing.B) {
+ b.StopTimer()
+ sel := DocB().Find("dd")
+ j := 0
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ sel.Eq(j)
+ if j++; j >= sel.Length() {
+ j = 0
+ }
+ }
+}
+
+func BenchmarkSlice(b *testing.B) {
+ b.StopTimer()
+ sel := DocB().Find("dd")
+ j := 0
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ sel.Slice(j, j+4)
+ if j++; j >= (sel.Length() - 4) {
+ j = 0
+ }
+ }
+}
+
+func BenchmarkGet(b *testing.B) {
+ b.StopTimer()
+ sel := DocB().Find("dd")
+ j := 0
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ sel.Get(j)
+ if j++; j >= sel.Length() {
+ j = 0
+ }
+ }
+}
+
+func BenchmarkIndex(b *testing.B) {
+ var j int
+
+ b.StopTimer()
+ sel := DocB().Find("#Main")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ j = sel.Index()
+ }
+ if j != 3 {
+ b.Fatalf("want 3, got %d", j)
+ }
+}
+
+func BenchmarkIndexSelector(b *testing.B) {
+ var j int
+
+ b.StopTimer()
+ sel := DocB().Find("#manual-nav dl dd:nth-child(1)")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ j = sel.IndexSelector("dd")
+ }
+ if j != 4 {
+ b.Fatalf("want 4, got %d", j)
+ }
+}
+
+func BenchmarkIndexOfNode(b *testing.B) {
+ var j int
+
+ b.StopTimer()
+ sel := DocB().Find("span a")
+ sel2 := DocB().Find("span a:nth-child(3)")
+ n := sel2.Get(0)
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ j = sel.IndexOfNode(n)
+ }
+ if j != 2 {
+ b.Fatalf("want 2, got %d", j)
+ }
+}
+
+func BenchmarkIndexOfSelection(b *testing.B) {
+ var j int
+ b.StopTimer()
+ sel := DocB().Find("span a")
+ sel2 := DocB().Find("span a:nth-child(3)")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ j = sel.IndexOfSelection(sel2)
+ }
+ if j != 2 {
+ b.Fatalf("want 2, got %d", j)
+ }
+}
diff --git a/vendor/github.com/PuerkitoBio/goquery/bench_example_test.go b/vendor/github.com/PuerkitoBio/goquery/bench_example_test.go
new file mode 100644
index 0000000..ba9ebe5
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/bench_example_test.go
@@ -0,0 +1,40 @@
+package goquery
+
+import (
+ "bytes"
+ "fmt"
+ "strconv"
+ "testing"
+)
+
+func BenchmarkMetalReviewExample(b *testing.B) {
+ var n int
+ var buf bytes.Buffer
+
+ b.StopTimer()
+ doc := loadDoc("metalreview.html")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ doc.Find(".slider-row:nth-child(1) .slider-item").Each(func(i int, s *Selection) {
+ var band, title string
+ var score float64
+ var e error
+
+ n++
+ // For each item found, get the band, title and score, and print it
+ band = s.Find("strong").Text()
+ title = s.Find("em").Text()
+ if score, e = strconv.ParseFloat(s.Find(".score").Text(), 64); e != nil {
+ // Not a valid float, ignore score
+ if n <= 4 {
+ buf.WriteString(fmt.Sprintf("Review %d: %s - %s.\n", i, band, title))
+ }
+ } else {
+ // Print all, including score
+ if n <= 4 {
+ buf.WriteString(fmt.Sprintf("Review %d: %s - %s (%2.1f).\n", i, band, title, score))
+ }
+ }
+ })
+ }
+}
diff --git a/vendor/github.com/PuerkitoBio/goquery/bench_expand_test.go b/vendor/github.com/PuerkitoBio/goquery/bench_expand_test.go
new file mode 100644
index 0000000..61f1947
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/bench_expand_test.go
@@ -0,0 +1,104 @@
+package goquery
+
+import (
+ "testing"
+)
+
+func BenchmarkAdd(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocB().Find("dd")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.Add("h2[title]").Length()
+ } else {
+ sel.Add("h2[title]")
+ }
+ }
+ if n != 43 {
+ b.Fatalf("want 43, got %d", n)
+ }
+}
+
+func BenchmarkAddSelection(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocB().Find("dd")
+ sel2 := DocB().Find("h2[title]")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.AddSelection(sel2).Length()
+ } else {
+ sel.AddSelection(sel2)
+ }
+ }
+ if n != 43 {
+ b.Fatalf("want 43, got %d", n)
+ }
+}
+
+func BenchmarkAddNodes(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocB().Find("dd")
+ sel2 := DocB().Find("h2[title]")
+ nodes := sel2.Nodes
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.AddNodes(nodes...).Length()
+ } else {
+ sel.AddNodes(nodes...)
+ }
+ }
+ if n != 43 {
+ b.Fatalf("want 43, got %d", n)
+ }
+}
+
+func BenchmarkAddNodesBig(b *testing.B) {
+ var n int
+
+ doc := DocW()
+ sel := doc.Find("li")
+ // make nodes > 1000
+ nodes := sel.Nodes
+ nodes = append(nodes, nodes...)
+ nodes = append(nodes, nodes...)
+ sel = doc.Find("xyz")
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.AddNodes(nodes...).Length()
+ } else {
+ sel.AddNodes(nodes...)
+ }
+ }
+ if n != 373 {
+ b.Fatalf("want 373, got %d", n)
+ }
+}
+
+func BenchmarkAndSelf(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocB().Find("dd").Parent()
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.AndSelf().Length()
+ } else {
+ sel.AndSelf()
+ }
+ }
+ if n != 44 {
+ b.Fatalf("want 44, got %d", n)
+ }
+}
diff --git a/vendor/github.com/PuerkitoBio/goquery/bench_filter_test.go b/vendor/github.com/PuerkitoBio/goquery/bench_filter_test.go
new file mode 100644
index 0000000..38e39f5
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/bench_filter_test.go
@@ -0,0 +1,236 @@
+package goquery
+
+import (
+ "testing"
+)
+
+func BenchmarkFilter(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.Filter(".toclevel-1").Length()
+ } else {
+ sel.Filter(".toclevel-1")
+ }
+ }
+ if n != 13 {
+ b.Fatalf("want 13, got %d", n)
+ }
+}
+
+func BenchmarkNot(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.Not(".toclevel-2").Length()
+ } else {
+ sel.Filter(".toclevel-2")
+ }
+ }
+ if n != 371 {
+ b.Fatalf("want 371, got %d", n)
+ }
+}
+
+func BenchmarkFilterFunction(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ f := func(i int, s *Selection) bool {
+ return len(s.Get(0).Attr) > 0
+ }
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.FilterFunction(f).Length()
+ } else {
+ sel.FilterFunction(f)
+ }
+ }
+ if n != 112 {
+ b.Fatalf("want 112, got %d", n)
+ }
+}
+
+func BenchmarkNotFunction(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ f := func(i int, s *Selection) bool {
+ return len(s.Get(0).Attr) > 0
+ }
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.NotFunction(f).Length()
+ } else {
+ sel.NotFunction(f)
+ }
+ }
+ if n != 261 {
+ b.Fatalf("want 261, got %d", n)
+ }
+}
+
+func BenchmarkFilterNodes(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ sel2 := DocW().Find(".toclevel-2")
+ nodes := sel2.Nodes
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.FilterNodes(nodes...).Length()
+ } else {
+ sel.FilterNodes(nodes...)
+ }
+ }
+ if n != 2 {
+ b.Fatalf("want 2, got %d", n)
+ }
+}
+
+func BenchmarkNotNodes(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ sel2 := DocW().Find(".toclevel-1")
+ nodes := sel2.Nodes
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.NotNodes(nodes...).Length()
+ } else {
+ sel.NotNodes(nodes...)
+ }
+ }
+ if n != 360 {
+ b.Fatalf("want 360, got %d", n)
+ }
+}
+
+func BenchmarkFilterSelection(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ sel2 := DocW().Find(".toclevel-2")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.FilterSelection(sel2).Length()
+ } else {
+ sel.FilterSelection(sel2)
+ }
+ }
+ if n != 2 {
+ b.Fatalf("want 2, got %d", n)
+ }
+}
+
+func BenchmarkNotSelection(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ sel2 := DocW().Find(".toclevel-1")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.NotSelection(sel2).Length()
+ } else {
+ sel.NotSelection(sel2)
+ }
+ }
+ if n != 360 {
+ b.Fatalf("want 360, got %d", n)
+ }
+}
+
+func BenchmarkHas(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.Has(".editsection").Length()
+ } else {
+ sel.Has(".editsection")
+ }
+ }
+ if n != 13 {
+ b.Fatalf("want 13, got %d", n)
+ }
+}
+
+func BenchmarkHasNodes(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ sel2 := DocW().Find(".tocnumber")
+ nodes := sel2.Nodes
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.HasNodes(nodes...).Length()
+ } else {
+ sel.HasNodes(nodes...)
+ }
+ }
+ if n != 15 {
+ b.Fatalf("want 15, got %d", n)
+ }
+}
+
+func BenchmarkHasSelection(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ sel2 := DocW().Find(".tocnumber")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.HasSelection(sel2).Length()
+ } else {
+ sel.HasSelection(sel2)
+ }
+ }
+ if n != 15 {
+ b.Fatalf("want 15, got %d", n)
+ }
+}
+
+func BenchmarkEnd(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li").Has(".tocnumber")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.End().Length()
+ } else {
+ sel.End()
+ }
+ }
+ if n != 373 {
+ b.Fatalf("want 373, got %d", n)
+ }
+}
diff --git a/vendor/github.com/PuerkitoBio/goquery/bench_iteration_test.go b/vendor/github.com/PuerkitoBio/goquery/bench_iteration_test.go
new file mode 100644
index 0000000..39445b0
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/bench_iteration_test.go
@@ -0,0 +1,68 @@
+package goquery
+
+import (
+ "testing"
+)
+
+func BenchmarkEach(b *testing.B) {
+ var tmp, n int
+
+ b.StopTimer()
+ sel := DocW().Find("td")
+ f := func(i int, s *Selection) {
+ tmp++
+ }
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ sel.Each(f)
+ if n == 0 {
+ n = tmp
+ }
+ }
+ if n != 59 {
+ b.Fatalf("want 59, got %d", n)
+ }
+}
+
+func BenchmarkMap(b *testing.B) {
+ var tmp, n int
+
+ b.StopTimer()
+ sel := DocW().Find("td")
+ f := func(i int, s *Selection) string {
+ tmp++
+ return string(tmp)
+ }
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ sel.Map(f)
+ if n == 0 {
+ n = tmp
+ }
+ }
+ if n != 59 {
+ b.Fatalf("want 59, got %d", n)
+ }
+}
+
+func BenchmarkEachWithBreak(b *testing.B) {
+ var tmp, n int
+
+ b.StopTimer()
+ sel := DocW().Find("td")
+ f := func(i int, s *Selection) bool {
+ tmp++
+ return tmp < 10
+ }
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ tmp = 0
+ sel.EachWithBreak(f)
+ if n == 0 {
+ n = tmp
+ }
+ }
+ if n != 10 {
+ b.Fatalf("want 10, got %d", n)
+ }
+}
diff --git a/vendor/github.com/PuerkitoBio/goquery/bench_property_test.go b/vendor/github.com/PuerkitoBio/goquery/bench_property_test.go
new file mode 100644
index 0000000..8acf5bf
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/bench_property_test.go
@@ -0,0 +1,51 @@
+package goquery
+
+import (
+ "testing"
+)
+
+func BenchmarkAttr(b *testing.B) {
+ var s string
+
+ b.StopTimer()
+ sel := DocW().Find("h1")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ s, _ = sel.Attr("id")
+ }
+ if s != "firstHeading" {
+ b.Fatalf("want firstHeading, got %q", s)
+ }
+}
+
+func BenchmarkText(b *testing.B) {
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ sel.Text()
+ }
+}
+
+func BenchmarkLength(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ n = sel.Length()
+ }
+ if n != 14 {
+ b.Fatalf("want 14, got %d", n)
+ }
+}
+
+func BenchmarkHtml(b *testing.B) {
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ sel.Html()
+ }
+}
diff --git a/vendor/github.com/PuerkitoBio/goquery/bench_query_test.go b/vendor/github.com/PuerkitoBio/goquery/bench_query_test.go
new file mode 100644
index 0000000..64fdbc4
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/bench_query_test.go
@@ -0,0 +1,111 @@
+package goquery
+
+import (
+ "testing"
+)
+
+func BenchmarkIs(b *testing.B) {
+ var y bool
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ y = sel.Is(".toclevel-2")
+ }
+ if !y {
+ b.Fatal("want true")
+ }
+}
+
+func BenchmarkIsPositional(b *testing.B) {
+ var y bool
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ y = sel.Is("li:nth-child(2)")
+ }
+ if !y {
+ b.Fatal("want true")
+ }
+}
+
+func BenchmarkIsFunction(b *testing.B) {
+ var y bool
+
+ b.StopTimer()
+ sel := DocW().Find(".toclevel-1")
+ f := func(i int, s *Selection) bool {
+ return i == 8
+ }
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ y = sel.IsFunction(f)
+ }
+ if !y {
+ b.Fatal("want true")
+ }
+}
+
+func BenchmarkIsSelection(b *testing.B) {
+ var y bool
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ sel2 := DocW().Find(".toclevel-2")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ y = sel.IsSelection(sel2)
+ }
+ if !y {
+ b.Fatal("want true")
+ }
+}
+
+func BenchmarkIsNodes(b *testing.B) {
+ var y bool
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ sel2 := DocW().Find(".toclevel-2")
+ nodes := sel2.Nodes
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ y = sel.IsNodes(nodes...)
+ }
+ if !y {
+ b.Fatal("want true")
+ }
+}
+
+func BenchmarkHasClass(b *testing.B) {
+ var y bool
+
+ b.StopTimer()
+ sel := DocW().Find("span")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ y = sel.HasClass("official")
+ }
+ if !y {
+ b.Fatal("want true")
+ }
+}
+
+func BenchmarkContains(b *testing.B) {
+ var y bool
+
+ b.StopTimer()
+ sel := DocW().Find("span.url")
+ sel2 := DocW().Find("a[rel=\"nofollow\"]")
+ node := sel2.Nodes[0]
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ y = sel.Contains(node)
+ }
+ if !y {
+ b.Fatal("want true")
+ }
+}
diff --git a/vendor/github.com/PuerkitoBio/goquery/bench_traversal_test.go b/vendor/github.com/PuerkitoBio/goquery/bench_traversal_test.go
new file mode 100644
index 0000000..de84bcd
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/bench_traversal_test.go
@@ -0,0 +1,802 @@
+package goquery
+
+import (
+ "testing"
+)
+
+func BenchmarkFind(b *testing.B) {
+ var n int
+
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = DocB().Find("dd").Length()
+
+ } else {
+ DocB().Find("dd")
+ }
+ }
+ if n != 41 {
+ b.Fatalf("want 41, got %d", n)
+ }
+}
+
+func BenchmarkFindWithinSelection(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("ul")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.Find("a[class]").Length()
+ } else {
+ sel.Find("a[class]")
+ }
+ }
+ if n != 39 {
+ b.Fatalf("want 39, got %d", n)
+ }
+}
+
+func BenchmarkFindSelection(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("ul")
+ sel2 := DocW().Find("span")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.FindSelection(sel2).Length()
+ } else {
+ sel.FindSelection(sel2)
+ }
+ }
+ if n != 73 {
+ b.Fatalf("want 73, got %d", n)
+ }
+}
+
+func BenchmarkFindNodes(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("ul")
+ sel2 := DocW().Find("span")
+ nodes := sel2.Nodes
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.FindNodes(nodes...).Length()
+ } else {
+ sel.FindNodes(nodes...)
+ }
+ }
+ if n != 73 {
+ b.Fatalf("want 73, got %d", n)
+ }
+}
+
+func BenchmarkContents(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find(".toclevel-1")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.Contents().Length()
+ } else {
+ sel.Contents()
+ }
+ }
+ if n != 16 {
+ b.Fatalf("want 16, got %d", n)
+ }
+}
+
+func BenchmarkContentsFiltered(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find(".toclevel-1")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.ContentsFiltered("a[href=\"#Examples\"]").Length()
+ } else {
+ sel.ContentsFiltered("a[href=\"#Examples\"]")
+ }
+ }
+ if n != 1 {
+ b.Fatalf("want 1, got %d", n)
+ }
+}
+
+func BenchmarkChildren(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find(".toclevel-2")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.Children().Length()
+ } else {
+ sel.Children()
+ }
+ }
+ if n != 2 {
+ b.Fatalf("want 2, got %d", n)
+ }
+}
+
+func BenchmarkChildrenFiltered(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("h3")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.ChildrenFiltered(".editsection").Length()
+ } else {
+ sel.ChildrenFiltered(".editsection")
+ }
+ }
+ if n != 2 {
+ b.Fatalf("want 2, got %d", n)
+ }
+}
+
+func BenchmarkParent(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.Parent().Length()
+ } else {
+ sel.Parent()
+ }
+ }
+ if n != 55 {
+ b.Fatalf("want 55, got %d", n)
+ }
+}
+
+func BenchmarkParentFiltered(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.ParentFiltered("ul[id]").Length()
+ } else {
+ sel.ParentFiltered("ul[id]")
+ }
+ }
+ if n != 4 {
+ b.Fatalf("want 4, got %d", n)
+ }
+}
+
+func BenchmarkParents(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("th a")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.Parents().Length()
+ } else {
+ sel.Parents()
+ }
+ }
+ if n != 73 {
+ b.Fatalf("want 73, got %d", n)
+ }
+}
+
+func BenchmarkParentsFiltered(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("th a")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.ParentsFiltered("tr").Length()
+ } else {
+ sel.ParentsFiltered("tr")
+ }
+ }
+ if n != 18 {
+ b.Fatalf("want 18, got %d", n)
+ }
+}
+
+func BenchmarkParentsUntil(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("th a")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.ParentsUntil("table").Length()
+ } else {
+ sel.ParentsUntil("table")
+ }
+ }
+ if n != 52 {
+ b.Fatalf("want 52, got %d", n)
+ }
+}
+
+func BenchmarkParentsUntilSelection(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("th a")
+ sel2 := DocW().Find("#content")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.ParentsUntilSelection(sel2).Length()
+ } else {
+ sel.ParentsUntilSelection(sel2)
+ }
+ }
+ if n != 70 {
+ b.Fatalf("want 70, got %d", n)
+ }
+}
+
+func BenchmarkParentsUntilNodes(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("th a")
+ sel2 := DocW().Find("#content")
+ nodes := sel2.Nodes
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.ParentsUntilNodes(nodes...).Length()
+ } else {
+ sel.ParentsUntilNodes(nodes...)
+ }
+ }
+ if n != 70 {
+ b.Fatalf("want 70, got %d", n)
+ }
+}
+
+func BenchmarkParentsFilteredUntil(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find(".toclevel-1 a")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.ParentsFilteredUntil(":nth-child(1)", "ul").Length()
+ } else {
+ sel.ParentsFilteredUntil(":nth-child(1)", "ul")
+ }
+ }
+ if n != 2 {
+ b.Fatalf("want 2, got %d", n)
+ }
+}
+
+func BenchmarkParentsFilteredUntilSelection(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find(".toclevel-1 a")
+ sel2 := DocW().Find("ul")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.ParentsFilteredUntilSelection(":nth-child(1)", sel2).Length()
+ } else {
+ sel.ParentsFilteredUntilSelection(":nth-child(1)", sel2)
+ }
+ }
+ if n != 2 {
+ b.Fatalf("want 2, got %d", n)
+ }
+}
+
+func BenchmarkParentsFilteredUntilNodes(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find(".toclevel-1 a")
+ sel2 := DocW().Find("ul")
+ nodes := sel2.Nodes
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.ParentsFilteredUntilNodes(":nth-child(1)", nodes...).Length()
+ } else {
+ sel.ParentsFilteredUntilNodes(":nth-child(1)", nodes...)
+ }
+ }
+ if n != 2 {
+ b.Fatalf("want 2, got %d", n)
+ }
+}
+
+func BenchmarkSiblings(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("ul li:nth-child(1)")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.Siblings().Length()
+ } else {
+ sel.Siblings()
+ }
+ }
+ if n != 293 {
+ b.Fatalf("want 293, got %d", n)
+ }
+}
+
+func BenchmarkSiblingsFiltered(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("ul li:nth-child(1)")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.SiblingsFiltered("[class]").Length()
+ } else {
+ sel.SiblingsFiltered("[class]")
+ }
+ }
+ if n != 46 {
+ b.Fatalf("want 46, got %d", n)
+ }
+}
+
+func BenchmarkNext(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li:nth-child(1)")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.Next().Length()
+ } else {
+ sel.Next()
+ }
+ }
+ if n != 49 {
+ b.Fatalf("want 49, got %d", n)
+ }
+}
+
+func BenchmarkNextFiltered(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li:nth-child(1)")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.NextFiltered("[class]").Length()
+ } else {
+ sel.NextFiltered("[class]")
+ }
+ }
+ if n != 6 {
+ b.Fatalf("want 6, got %d", n)
+ }
+}
+
+func BenchmarkNextAll(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li:nth-child(3)")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.NextAll().Length()
+ } else {
+ sel.NextAll()
+ }
+ }
+ if n != 234 {
+ b.Fatalf("want 234, got %d", n)
+ }
+}
+
+func BenchmarkNextAllFiltered(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li:nth-child(3)")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.NextAllFiltered("[class]").Length()
+ } else {
+ sel.NextAllFiltered("[class]")
+ }
+ }
+ if n != 33 {
+ b.Fatalf("want 33, got %d", n)
+ }
+}
+
+func BenchmarkPrev(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li:last-child")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.Prev().Length()
+ } else {
+ sel.Prev()
+ }
+ }
+ if n != 49 {
+ b.Fatalf("want 49, got %d", n)
+ }
+}
+
+func BenchmarkPrevFiltered(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li:last-child")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.PrevFiltered("[class]").Length()
+ } else {
+ sel.PrevFiltered("[class]")
+ }
+ }
+ // There is one more Prev li with a class, compared to Next li with a class
+ // (confirmed by looking at the HTML, this is ok)
+ if n != 7 {
+ b.Fatalf("want 7, got %d", n)
+ }
+}
+
+func BenchmarkPrevAll(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li:nth-child(4)")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.PrevAll().Length()
+ } else {
+ sel.PrevAll()
+ }
+ }
+ if n != 78 {
+ b.Fatalf("want 78, got %d", n)
+ }
+}
+
+func BenchmarkPrevAllFiltered(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li:nth-child(4)")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.PrevAllFiltered("[class]").Length()
+ } else {
+ sel.PrevAllFiltered("[class]")
+ }
+ }
+ if n != 6 {
+ b.Fatalf("want 6, got %d", n)
+ }
+}
+
+func BenchmarkNextUntil(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li:first-child")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.NextUntil(":nth-child(4)").Length()
+ } else {
+ sel.NextUntil(":nth-child(4)")
+ }
+ }
+ if n != 84 {
+ b.Fatalf("want 84, got %d", n)
+ }
+}
+
+func BenchmarkNextUntilSelection(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ sel2 := DocW().Find("ul")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.NextUntilSelection(sel2).Length()
+ } else {
+ sel.NextUntilSelection(sel2)
+ }
+ }
+ if n != 42 {
+ b.Fatalf("want 42, got %d", n)
+ }
+}
+
+func BenchmarkNextUntilNodes(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ sel2 := DocW().Find("p")
+ nodes := sel2.Nodes
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.NextUntilNodes(nodes...).Length()
+ } else {
+ sel.NextUntilNodes(nodes...)
+ }
+ }
+ if n != 12 {
+ b.Fatalf("want 12, got %d", n)
+ }
+}
+
+func BenchmarkPrevUntil(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("li:last-child")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.PrevUntil(":nth-child(4)").Length()
+ } else {
+ sel.PrevUntil(":nth-child(4)")
+ }
+ }
+ if n != 238 {
+ b.Fatalf("want 238, got %d", n)
+ }
+}
+
+func BenchmarkPrevUntilSelection(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ sel2 := DocW().Find("ul")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.PrevUntilSelection(sel2).Length()
+ } else {
+ sel.PrevUntilSelection(sel2)
+ }
+ }
+ if n != 49 {
+ b.Fatalf("want 49, got %d", n)
+ }
+}
+
+func BenchmarkPrevUntilNodes(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ sel2 := DocW().Find("p")
+ nodes := sel2.Nodes
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.PrevUntilNodes(nodes...).Length()
+ } else {
+ sel.PrevUntilNodes(nodes...)
+ }
+ }
+ if n != 11 {
+ b.Fatalf("want 11, got %d", n)
+ }
+}
+
+func BenchmarkNextFilteredUntil(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.NextFilteredUntil("p", "div").Length()
+ } else {
+ sel.NextFilteredUntil("p", "div")
+ }
+ }
+ if n != 22 {
+ b.Fatalf("want 22, got %d", n)
+ }
+}
+
+func BenchmarkNextFilteredUntilSelection(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ sel2 := DocW().Find("div")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.NextFilteredUntilSelection("p", sel2).Length()
+ } else {
+ sel.NextFilteredUntilSelection("p", sel2)
+ }
+ }
+ if n != 22 {
+ b.Fatalf("want 22, got %d", n)
+ }
+}
+
+func BenchmarkNextFilteredUntilNodes(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ sel2 := DocW().Find("div")
+ nodes := sel2.Nodes
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.NextFilteredUntilNodes("p", nodes...).Length()
+ } else {
+ sel.NextFilteredUntilNodes("p", nodes...)
+ }
+ }
+ if n != 22 {
+ b.Fatalf("want 22, got %d", n)
+ }
+}
+
+func BenchmarkPrevFilteredUntil(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.PrevFilteredUntil("p", "div").Length()
+ } else {
+ sel.PrevFilteredUntil("p", "div")
+ }
+ }
+ if n != 20 {
+ b.Fatalf("want 20, got %d", n)
+ }
+}
+
+func BenchmarkPrevFilteredUntilSelection(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ sel2 := DocW().Find("div")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.PrevFilteredUntilSelection("p", sel2).Length()
+ } else {
+ sel.PrevFilteredUntilSelection("p", sel2)
+ }
+ }
+ if n != 20 {
+ b.Fatalf("want 20, got %d", n)
+ }
+}
+
+func BenchmarkPrevFilteredUntilNodes(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := DocW().Find("h2")
+ sel2 := DocW().Find("div")
+ nodes := sel2.Nodes
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.PrevFilteredUntilNodes("p", nodes...).Length()
+ } else {
+ sel.PrevFilteredUntilNodes("p", nodes...)
+ }
+ }
+ if n != 20 {
+ b.Fatalf("want 20, got %d", n)
+ }
+}
+
+func BenchmarkClosest(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := Doc().Find(".container-fluid")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.Closest(".pvk-content").Length()
+ } else {
+ sel.Closest(".pvk-content")
+ }
+ }
+ if n != 2 {
+ b.Fatalf("want 2, got %d", n)
+ }
+}
+
+func BenchmarkClosestSelection(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := Doc().Find(".container-fluid")
+ sel2 := Doc().Find(".pvk-content")
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.ClosestSelection(sel2).Length()
+ } else {
+ sel.ClosestSelection(sel2)
+ }
+ }
+ if n != 2 {
+ b.Fatalf("want 2, got %d", n)
+ }
+}
+
+func BenchmarkClosestNodes(b *testing.B) {
+ var n int
+
+ b.StopTimer()
+ sel := Doc().Find(".container-fluid")
+ nodes := Doc().Find(".pvk-content").Nodes
+ b.StartTimer()
+ for i := 0; i < b.N; i++ {
+ if n == 0 {
+ n = sel.ClosestNodes(nodes...).Length()
+ } else {
+ sel.ClosestNodes(nodes...)
+ }
+ }
+ if n != 2 {
+ b.Fatalf("want 2, got %d", n)
+ }
+}
diff --git a/vendor/github.com/PuerkitoBio/goquery/doc.go b/vendor/github.com/PuerkitoBio/goquery/doc.go
new file mode 100644
index 0000000..71146a7
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/doc.go
@@ -0,0 +1,123 @@
+// Copyright (c) 2012-2016, Martin Angers & Contributors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation and/or
+// other materials provided with the distribution.
+// * Neither the name of the author nor the names of its contributors may be used to
+// endorse or promote products derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
+// OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
+// WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/*
+Package goquery implements features similar to jQuery, including the chainable
+syntax, to manipulate and query an HTML document.
+
+It brings a syntax and a set of features similar to jQuery to the Go language.
+It is based on Go's net/html package and the CSS Selector library cascadia.
+Since the net/html parser returns nodes, and not a full-featured DOM
+tree, jQuery's stateful manipulation functions (like height(), css(), detach())
+have been left off.
+
+Also, because the net/html parser requires UTF-8 encoding, so does goquery: it is
+the caller's responsibility to ensure that the source document provides UTF-8 encoded HTML.
+See the repository's wiki for various options on how to do this.
+
+Syntax-wise, it is as close as possible to jQuery, with the same method names when
+possible, and that warm and fuzzy chainable interface. jQuery being the
+ultra-popular library that it is, writing a similar HTML-manipulating
+library was better to follow its API than to start anew (in the same spirit as
+Go's fmt package), even though some of its methods are less than intuitive (looking
+at you, index()...).
+
+It is hosted on GitHub, along with additional documentation in the README.md
+file: https://github.com/puerkitobio/goquery
+
+Please note that because of the net/html dependency, goquery requires Go1.1+.
+
+The various methods are split into files based on the category of behavior.
+The three dots (...) indicate that various "overloads" are available.
+
+* array.go : array-like positional manipulation of the selection.
+ - Eq()
+ - First()
+ - Get()
+ - Index...()
+ - Last()
+ - Slice()
+
+* expand.go : methods that expand or augment the selection's set.
+ - Add...()
+ - AndSelf()
+ - Union(), which is an alias for AddSelection()
+
+* filter.go : filtering methods, that reduce the selection's set.
+ - End()
+ - Filter...()
+ - Has...()
+ - Intersection(), which is an alias of FilterSelection()
+ - Not...()
+
+* iteration.go : methods to loop over the selection's nodes.
+ - Each()
+ - EachWithBreak()
+ - Map()
+
+* manipulation.go : methods for modifying the document
+ - After...()
+ - Append...()
+ - Before...()
+ - Clone()
+ - Empty()
+ - Prepend...()
+ - Remove...()
+ - ReplaceWith...()
+ - Unwrap()
+ - Wrap...()
+ - WrapAll...()
+ - WrapInner...()
+
+* property.go : methods that inspect and get the node's properties values.
+ - Attr*(), RemoveAttr(), SetAttr()
+ - AddClass(), HasClass(), RemoveClass(), ToggleClass()
+ - Html()
+ - Length()
+ - Size(), which is an alias for Length()
+ - Text()
+
+* query.go : methods that query, or reflect, a node's identity.
+ - Contains()
+ - Is...()
+
+* traversal.go : methods to traverse the HTML document tree.
+ - Children...()
+ - Contents()
+ - Find...()
+ - Next...()
+ - Parent[s]...()
+ - Prev...()
+ - Siblings...()
+
+* type.go : definition of the types exposed by goquery.
+ - Document
+ - Selection
+ - Matcher
+
+* utilities.go : definition of helper functions (and not methods on a *Selection)
+that are not part of jQuery, but are useful to goquery.
+ - NodeName
+ - OuterHtml
+*/
+package goquery
diff --git a/vendor/github.com/PuerkitoBio/goquery/example_test.go b/vendor/github.com/PuerkitoBio/goquery/example_test.go
new file mode 100644
index 0000000..bc97f05
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/example_test.go
@@ -0,0 +1,82 @@
+package goquery_test
+
+import (
+ "fmt"
+ "log"
+ "net/http"
+ "os"
+ "strings"
+
+ "github.com/PuerkitoBio/goquery"
+)
+
+// This example scrapes the reviews shown on the home page of metalsucks.net.
+func Example() {
+ // Request the HTML page.
+ res, err := http.Get("http://metalsucks.net")
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer res.Body.Close()
+ if res.StatusCode != 200 {
+ log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
+ }
+
+ // Load the HTML document
+ doc, err := goquery.NewDocumentFromReader(res.Body)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ // Find the review items
+ doc.Find(".sidebar-reviews article .content-block").Each(func(i int, s *goquery.Selection) {
+ // For each item found, get the band and title
+ band := s.Find("a").Text()
+ title := s.Find("i").Text()
+ fmt.Printf("Review %d: %s - %s\n", i, band, title)
+ })
+ // To see the output of the Example while running the test suite (go test), simply
+ // remove the leading "x" before Output on the next line. This will cause the
+ // example to fail (all the "real" tests should pass).
+
+ // xOutput: voluntarily fail the Example output.
+}
+
+// This example shows how to use NewDocumentFromReader from a file.
+func ExampleNewDocumentFromReader_file() {
+ // create from a file
+ f, err := os.Open("some/file.html")
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer f.Close()
+ doc, err := goquery.NewDocumentFromReader(f)
+ if err != nil {
+ log.Fatal(err)
+ }
+ // use the goquery document...
+ _ = doc.Find("h1")
+}
+
+// This example shows how to use NewDocumentFromReader from a string.
+func ExampleNewDocumentFromReader_string() {
+ // create from a string
+ data := `
+
+
+ Some text
` + d, err := NewDocumentFromReader(strings.NewReader(src)) + if err != nil { + t.Fatal(err) + } + txt := d.Find("p").Text() + ix := strings.Index(txt, "\u00a0") + if ix != 4 { + t.Errorf("Text: expected a non-breaking space at index 4, got %d", ix) + } + + h, err := d.Find("p").Html() + if err != nil { + t.Fatal(err) + } + ix = strings.Index(h, "\u00a0") + if ix != 4 { + t.Errorf("Html: expected a non-breaking space at index 4, got %d", ix) + } +} + +func TestAddClass(t *testing.T) { + sel := Doc2Clone().Find("#main") + sel.AddClass("main main main") + + // Make sure that class was only added once + if a, ok := sel.Attr("class"); !ok || a != "main" { + t.Error("Expected #main to have class main") + } +} + +func TestAddClassSimilar(t *testing.T) { + sel := Doc2Clone().Find("#nf5") + sel.AddClass("odd") + + assertClass(t, sel, "odd") + assertClass(t, sel, "odder") + printSel(t, sel.Parent()) +} + +func TestAddEmptyClass(t *testing.T) { + sel := Doc2Clone().Find("#main") + sel.AddClass("") + + // Make sure that class was only added once + if a, ok := sel.Attr("class"); ok { + t.Errorf("Expected #main to not to have a class, have: %s", a) + } +} + +func TestAddClasses(t *testing.T) { + sel := Doc2Clone().Find("#main") + sel.AddClass("a b") + + // Make sure that class was only added once + if !sel.HasClass("a") || !sel.HasClass("b") { + t.Errorf("#main does not have classes") + } +} + +func TestHasClass(t *testing.T) { + sel := Doc().Find("div") + if !sel.HasClass("span12") { + t.Error("Expected at least one div to have class span12.") + } +} + +func TestHasClassNone(t *testing.T) { + sel := Doc().Find("h2") + if sel.HasClass("toto") { + t.Error("Expected h1 to have no class.") + } +} + +func TestHasClassNotFirst(t *testing.T) { + sel := Doc().Find(".alert") + if !sel.HasClass("alert-error") { + t.Error("Expected .alert to also have class .alert-error.") + } +} + +func TestRemoveClass(t *testing.T) { + sel := Doc2Clone().Find("#nf1") + sel.RemoveClass("one row") + + if !sel.HasClass("even") || sel.HasClass("one") || sel.HasClass("row") { + classes, _ := sel.Attr("class") + t.Error("Expected #nf1 to have class even, has ", classes) + } +} + +func TestRemoveClassSimilar(t *testing.T) { + sel := Doc2Clone().Find("#nf5, #nf6") + assertLength(t, sel.Nodes, 2) + + sel.RemoveClass("odd") + assertClass(t, sel.Eq(0), "odder") + printSel(t, sel) +} + +func TestRemoveAllClasses(t *testing.T) { + sel := Doc2Clone().Find("#nf1") + sel.RemoveClass() + + if a, ok := sel.Attr("class"); ok { + t.Error("All classes were not removed, has ", a) + } + + sel = Doc2Clone().Find("#main") + sel.RemoveClass() + if a, ok := sel.Attr("class"); ok { + t.Error("All classes were not removed, has ", a) + } +} + +func TestToggleClass(t *testing.T) { + sel := Doc2Clone().Find("#nf1") + + sel.ToggleClass("one") + if sel.HasClass("one") { + t.Error("Expected #nf1 to not have class one") + } + + sel.ToggleClass("one") + if !sel.HasClass("one") { + t.Error("Expected #nf1 to have class one") + } + + sel.ToggleClass("one even row") + if a, ok := sel.Attr("class"); ok { + t.Errorf("Expected #nf1 to have no classes, have %q", a) + } +} diff --git a/vendor/github.com/PuerkitoBio/goquery/query.go b/vendor/github.com/PuerkitoBio/goquery/query.go new file mode 100644 index 0000000..fe86bf0 --- /dev/null +++ b/vendor/github.com/PuerkitoBio/goquery/query.go @@ -0,0 +1,49 @@ +package goquery + +import "golang.org/x/net/html" + +// Is checks the current matched set of elements against a selector and +// returns true if at least one of these elements matches. +func (s *Selection) Is(selector string) bool { + return s.IsMatcher(compileMatcher(selector)) +} + +// IsMatcher checks the current matched set of elements against a matcher and +// returns true if at least one of these elements matches. +func (s *Selection) IsMatcher(m Matcher) bool { + if len(s.Nodes) > 0 { + if len(s.Nodes) == 1 { + return m.Match(s.Nodes[0]) + } + return len(m.Filter(s.Nodes)) > 0 + } + + return false +} + +// IsFunction checks the current matched set of elements against a predicate and +// returns true if at least one of these elements matches. +func (s *Selection) IsFunction(f func(int, *Selection) bool) bool { + return s.FilterFunction(f).Length() > 0 +} + +// IsSelection checks the current matched set of elements against a Selection object +// and returns true if at least one of these elements matches. +func (s *Selection) IsSelection(sel *Selection) bool { + return s.FilterSelection(sel).Length() > 0 +} + +// IsNodes checks the current matched set of elements against the specified nodes +// and returns true if at least one of these elements matches. +func (s *Selection) IsNodes(nodes ...*html.Node) bool { + return s.FilterNodes(nodes...).Length() > 0 +} + +// Contains returns true if the specified Node is within, +// at any depth, one of the nodes in the Selection object. +// It is NOT inclusive, to behave like jQuery's implementation, and +// unlike Javascript's .contains, so if the contained +// node is itself in the selection, it returns false. +func (s *Selection) Contains(n *html.Node) bool { + return sliceContains(s.Nodes, n) +} diff --git a/vendor/github.com/PuerkitoBio/goquery/query_test.go b/vendor/github.com/PuerkitoBio/goquery/query_test.go new file mode 100644 index 0000000..54b2a2e --- /dev/null +++ b/vendor/github.com/PuerkitoBio/goquery/query_test.go @@ -0,0 +1,103 @@ +package goquery + +import ( + "testing" +) + +func TestIs(t *testing.T) { + sel := Doc().Find(".footer p:nth-child(1)") + if !sel.Is("p") { + t.Error("Expected .footer p:nth-child(1) to be p.") + } +} + +func TestIsInvalid(t *testing.T) { + sel := Doc().Find(".footer p:nth-child(1)") + if sel.Is("") { + t.Error("Is should not succeed with invalid selector string") + } +} + +func TestIsPositional(t *testing.T) { + sel := Doc().Find(".footer p:nth-child(2)") + if !sel.Is("p:nth-child(2)") { + t.Error("Expected .footer p:nth-child(2) to be p:nth-child(2).") + } +} + +func TestIsPositionalNot(t *testing.T) { + sel := Doc().Find(".footer p:nth-child(1)") + if sel.Is("p:nth-child(2)") { + t.Error("Expected .footer p:nth-child(1) NOT to be p:nth-child(2).") + } +} + +func TestIsFunction(t *testing.T) { + ok := Doc().Find("div").IsFunction(func(i int, s *Selection) bool { + return s.HasClass("container-fluid") + }) + + if !ok { + t.Error("Expected some div to have a container-fluid class.") + } +} + +func TestIsFunctionRollback(t *testing.T) { + ok := Doc().Find("div").IsFunction(func(i int, s *Selection) bool { + return s.HasClass("container-fluid") + }) + + if !ok { + t.Error("Expected some div to have a container-fluid class.") + } +} + +func TestIsSelection(t *testing.T) { + sel := Doc().Find("div") + sel2 := Doc().Find(".pvk-gutter") + + if !sel.IsSelection(sel2) { + t.Error("Expected some div to have a pvk-gutter class.") + } +} + +func TestIsSelectionNot(t *testing.T) { + sel := Doc().Find("div") + sel2 := Doc().Find("a") + + if sel.IsSelection(sel2) { + t.Error("Expected some div NOT to be an anchor.") + } +} + +func TestIsNodes(t *testing.T) { + sel := Doc().Find("div") + sel2 := Doc().Find(".footer") + + if !sel.IsNodes(sel2.Nodes[0]) { + t.Error("Expected some div to have a footer class.") + } +} + +func TestDocContains(t *testing.T) { + sel := Doc().Find("h1") + if !Doc().Contains(sel.Nodes[0]) { + t.Error("Expected document to contain H1 tag.") + } +} + +func TestSelContains(t *testing.T) { + sel := Doc().Find(".row-fluid") + sel2 := Doc().Find("a[ng-click]") + if !sel.Contains(sel2.Nodes[0]) { + t.Error("Expected .row-fluid to contain a[ng-click] tag.") + } +} + +func TestSelNotContains(t *testing.T) { + sel := Doc().Find("a.link") + sel2 := Doc().Find("span") + if sel.Contains(sel2.Nodes[0]) { + t.Error("Expected a.link to NOT contain span tag.") + } +} diff --git a/vendor/github.com/PuerkitoBio/goquery/testdata/gotesting.html b/vendor/github.com/PuerkitoBio/goquery/testdata/gotesting.html new file mode 100644 index 0000000..ba5348f --- /dev/null +++ b/vendor/github.com/PuerkitoBio/goquery/testdata/gotesting.html @@ -0,0 +1,855 @@ + + + + + +| Name | ++ | Synopsis | +
|---|---|---|
| .. | +||
| iotest | ++ | Package iotest implements Readers and Writers useful mainly for testing. | +
| quick | ++ | Package quick implements utility functions to help with black box testing. | +
+ |
+This article's lead section may not adequately summarize all of its contents. (March 2012) | +
 |
+|
| Paradigm(s) | +compiled, concurrent, imperative, structured | +
|---|---|
| Appeared in | +2009 | +
| Designed by | +Robert Griesemer +Rob Pike +Ken Thompson |
+
| Developer | +Google Inc. | +
| Stable release | +version 1.0.2[1] (14 June 2012) | +
| Typing discipline | +strong, static | +
| Major implementations | +gc (8g, 6g, 5g), gccgo | +
| Influenced by | +C, Limbo, Modula, Newsqueak, Oberon, Pascal,[2] Python | +
| OS | +Linux, Mac OS X, FreeBSD, OpenBSD, MS Windows, Plan 9[3] | +
| License | +BSD-style[4] + Patent grant[5] | +
| Usual filename extensions | +.go | +
| Website | +golang.org | +
Go is a compiled, garbage-collected, concurrent programming language developed by Google Inc.[6]
+The initial design of Go was started in September 2007 by Robert Griesemer, Rob Pike, and Ken Thompson.[2] Go was officially announced in November 2009. In May 2010, Rob Pike publicly stated that Go was being used "for real stuff" at Google.[7] Go's "gc" compiler targets the Linux, Mac OS X, FreeBSD, OpenBSD, Plan 9, and Microsoft Windows operating systems and the i386, amd64, and ARM processor architectures.[8]
+|
+
+
+
+Contents+ |
+
Go aims to provide the efficiency of a statically typed compiled language with the ease of programming of a dynamic language.[9] Other goals include:
+The syntax of Go is broadly similar to that of C: blocks of code are surrounded with curly braces; common control flow structures include for, switch, and if. Unlike C, line-ending semicolons are optional, variable declarations are written differently and are usually optional, type conversions must be made explicit, and new go and select control keywords have been introduced to support concurrent programming. New built-in types include maps, Unicode strings, array slices, and channels for inter-thread communication.
Go is designed for exceptionally fast compiling times, even on modest hardware.[10] The language requires garbage collection. Certain concurrency-related structural conventions of Go (channels and alternative channel inputs) are borrowed from Tony Hoare's CSP. Unlike previous concurrent programming languages such as occam or Limbo, Go does not provide any built-in notion of safe or verifiable concurrency.[11]
+Of features found in C++ or Java, Go does not include type inheritance, generic programming, assertions, method overloading, or pointer arithmetic.[2] Of these, the Go authors express an openness to generic programming, explicitly argue against assertions and pointer arithmetic, while defending the choice to omit type inheritance as giving a more useful language, encouraging heavy use of interfaces instead.[2] Initially, the language did not include exception handling, but in March 2010 a mechanism known as panic/recover was implemented to handle exceptional errors while avoiding some of the problems the Go authors find with exceptions.[12][13]
Go allows a programmer to write functions that can operate on inputs of arbitrary type, provided that the type implements the functions defined by a given interface.
+Unlike Java, the interfaces a type supports do not need to be specified at the point at which the type is defined, and Go interfaces do not participate in a type hierarchy. A Go interface is best described as a set of methods, each identified by a name and signature. A type is considered to implement an interface if all the required methods have been defined for that type. An interface can be declared to "embed" other interfaces, meaning the declared interface includes the methods defined in the other interfaces.[11]
+Unlike Java, the in-memory representation of an object does not contain a pointer to a virtual method table. Instead a value of interface type is implemented as a pair of a pointer to the object, and a pointer to a dictionary containing implementations of the interface methods for that type.
+Consider the following example:
++type Sequence []int + +func (s Sequence) Len() int { + return len(s) +} + +type HasLength interface { + Len() int +} + +func Foo (o HasLength) { + ... +} +
These four definitions could have been placed in separate files, in different parts of the program. Notably, the programmer who defined the Sequence type did not need to declare that the type implemented HasLength, and the person who implemented the Len method for Sequence did not need to specify that this method was part of HasLength.
Visibility of structures, structure fields, variables, constants, methods, top-level types and functions outside their defining package is defined implicitly according to the capitalization of their identifier.[14]
+Go provides goroutines, small lightweight threads; the name alludes to coroutines. Goroutines are created with the go statement from anonymous or named functions.
Goroutines are executed in parallel with other goroutines, including their caller. They do not necessarily run in separate threads, but a group of goroutines are multiplexed onto multiple threads — execution control is moved between them by blocking them when sending or receiving messages over channels.
+There are currently two Go compilers:
+Both compilers work on Unix-like systems, and a port to Microsoft Windows of the gc compiler and runtime have been integrated in the main distribution. Most of the standard libraries also work on Windows.
+There is also an unmaintained "tiny" runtime environment that allows Go programs to run on bare hardware.[17]
+The following is a Hello world program in Go:
++package main + +import "fmt" + +func main() { + fmt.Println("Hello, World") +} +
Go's automatic semicolon insertion feature requires that opening braces not be placed on their own lines, and this is thus the preferred brace style; the examples shown comply with this style.[18]
+Example illustrating how to write a program like the Unix echo command in Go:[19]
++package main + +import ( + "os" + "flag" // command line option parser +) + +var omitNewline = flag.Bool("n", false, "don't print final newline") + +const ( + Space = " " + Newline = "\n" +) + +func main() { + flag.Parse() // Scans the arg list and sets up flags + var s string + for i := 0; i < flag.NArg(); i++ { + if i > 0 { + s += Space + } + s += flag.Arg(i) + } + if !*omitNewline { + s += Newline + } + os.Stdout.WriteString(s) +} +
Go's initial release led to much discussion.
+Michele Simionato wrote in an article for artima.com:[20]
+++Here I just wanted to point out the design choices about interfaces and inheritance. Such ideas are not new and it is a shame that no popular language has followed such particular route in the design space. I hope Go will become popular; if not, I hope such ideas will finally enter in a popular language, we are already 10 or 20 years too late :-(+
Dave Astels at Engine Yard wrote:[21]
+++Go is extremely easy to dive into. There are a minimal number of fundamental language concepts and the syntax is clean and designed to be clear and unambiguous. Go is still experimental and still a little rough around the edges.+
Ars Technica interviewed Rob Pike, one of the authors of Go, and asked why a new language was needed. He replied that:[22]
+++It wasn't enough to just add features to existing programming languages, because sometimes you can get more in the long run by taking things away. They wanted to start from scratch and rethink everything. ... [But they did not want] to deviate too much from what developers already knew because they wanted to avoid alienating Go's target audience.+
Go was in 15th place on the TIOBE Programming Community Index of programming language popularity in its first year, 2009,[citation needed] surpassing established languages like Pascal. As of March 2012[update], it ranked 66th in the index.[23]
+Bruce Eckel stated:[24]
+++The complexity of C++ (even more complexity has been added in the new C++), and the resulting impact on productivity, is no longer justified. All the hoops that the C++ programmer had to jump through in order to use a C-compatible language make no sense anymore -- they're just a waste of time and effort. Now, Go makes much more sense for the class of problems that C++ was originally intended to solve.+
On the day of the general release of the language, Francis McCabe, developer of the Go! programming language (note the exclamation point), requested a name change of Google's language to prevent confusion with his language.[25] The issue was closed by a Google developer on 12 October 2010 with the custom status "Unfortunate", with a comment that "there are many computing products and services named Go. In the 11 months since our release, there has been minimal confusion of the two languages."[26]
+
+
|
+|||||||||||||||||
+
|
+||||||||||||||
+
+ Serpentine Path
+ Serpentine Path
+
+
+
+ Hunter's Ground
+ No God But the Wild
+
+
+
+ Blut Aus Nord
+ 777 - Cosmosophy
+ Black
+
+
+ Ufomammut
+ Oro: Opus Alter
+ Doom
+
+
+ Resurgency
+ False Enlightenment
+ Death
+
+
+ Krallice
+ Years Past Matter
+ Black
+
+
+ Murder Construct
+ Results
+ Grindcore
+
+
+ Grave
+ Endless Procession of Souls
+ Death
+
+
+ Master
+ The New Elite
+ Death
+
+ Prove your point. +
+
`
+ img2 := ``
+ cases := []struct {
+ s string
+ l int
+ }{
+ {s: img1 + img2, l: 2},
+ {s: img1, l: 1},
+ {s: img2, l: 1},
+ }
+ for _, c := range cases {
+ doc, err := NewDocumentFromReader(strings.NewReader(c.s))
+ if err != nil {
+ t.Fatal(err)
+ }
+ sel := doc.Find("img[src]")
+ assertLength(t, sel.Nodes, c.l)
+ }
+}
diff --git a/vendor/github.com/PuerkitoBio/goquery/type.go b/vendor/github.com/PuerkitoBio/goquery/type.go
new file mode 100644
index 0000000..6ad51db
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/type.go
@@ -0,0 +1,141 @@
+package goquery
+
+import (
+ "errors"
+ "io"
+ "net/http"
+ "net/url"
+
+ "github.com/andybalholm/cascadia"
+
+ "golang.org/x/net/html"
+)
+
+// Document represents an HTML document to be manipulated. Unlike jQuery, which
+// is loaded as part of a DOM document, and thus acts upon its containing
+// document, GoQuery doesn't know which HTML document to act upon. So it needs
+// to be told, and that's what the Document class is for. It holds the root
+// document node to manipulate, and can make selections on this document.
+type Document struct {
+ *Selection
+ Url *url.URL
+ rootNode *html.Node
+}
+
+// NewDocumentFromNode is a Document constructor that takes a root html Node
+// as argument.
+func NewDocumentFromNode(root *html.Node) *Document {
+ return newDocument(root, nil)
+}
+
+// NewDocument is a Document constructor that takes a string URL as argument.
+// It loads the specified document, parses it, and stores the root Document
+// node, ready to be manipulated.
+//
+// Deprecated: Use the net/http standard library package to make the request
+// and validate the response before calling goquery.NewDocumentFromReader
+// with the response's body.
+func NewDocument(url string) (*Document, error) {
+ // Load the URL
+ res, e := http.Get(url)
+ if e != nil {
+ return nil, e
+ }
+ return NewDocumentFromResponse(res)
+}
+
+// NewDocumentFromReader returns a Document from an io.Reader.
+// It returns an error as second value if the reader's data cannot be parsed
+// as html. It does not check if the reader is also an io.Closer, the
+// provided reader is never closed by this call. It is the responsibility
+// of the caller to close it if required.
+func NewDocumentFromReader(r io.Reader) (*Document, error) {
+ root, e := html.Parse(r)
+ if e != nil {
+ return nil, e
+ }
+ return newDocument(root, nil), nil
+}
+
+// NewDocumentFromResponse is another Document constructor that takes an http response as argument.
+// It loads the specified response's document, parses it, and stores the root Document
+// node, ready to be manipulated. The response's body is closed on return.
+//
+// Deprecated: Use goquery.NewDocumentFromReader with the response's body.
+func NewDocumentFromResponse(res *http.Response) (*Document, error) {
+ if res == nil {
+ return nil, errors.New("Response is nil")
+ }
+ defer res.Body.Close()
+ if res.Request == nil {
+ return nil, errors.New("Response.Request is nil")
+ }
+
+ // Parse the HTML into nodes
+ root, e := html.Parse(res.Body)
+ if e != nil {
+ return nil, e
+ }
+
+ // Create and fill the document
+ return newDocument(root, res.Request.URL), nil
+}
+
+// CloneDocument creates a deep-clone of a document.
+func CloneDocument(doc *Document) *Document {
+ return newDocument(cloneNode(doc.rootNode), doc.Url)
+}
+
+// Private constructor, make sure all fields are correctly filled.
+func newDocument(root *html.Node, url *url.URL) *Document {
+ // Create and fill the document
+ d := &Document{nil, url, root}
+ d.Selection = newSingleSelection(root, d)
+ return d
+}
+
+// Selection represents a collection of nodes matching some criteria. The
+// initial Selection can be created by using Document.Find, and then
+// manipulated using the jQuery-like chainable syntax and methods.
+type Selection struct {
+ Nodes []*html.Node
+ document *Document
+ prevSel *Selection
+}
+
+// Helper constructor to create an empty selection
+func newEmptySelection(doc *Document) *Selection {
+ return &Selection{nil, doc, nil}
+}
+
+// Helper constructor to create a selection of only one node
+func newSingleSelection(node *html.Node, doc *Document) *Selection {
+ return &Selection{[]*html.Node{node}, doc, nil}
+}
+
+// Matcher is an interface that defines the methods to match
+// HTML nodes against a compiled selector string. Cascadia's
+// Selector implements this interface.
+type Matcher interface {
+ Match(*html.Node) bool
+ MatchAll(*html.Node) []*html.Node
+ Filter([]*html.Node) []*html.Node
+}
+
+// compileMatcher compiles the selector string s and returns
+// the corresponding Matcher. If s is an invalid selector string,
+// it returns a Matcher that fails all matches.
+func compileMatcher(s string) Matcher {
+ cs, err := cascadia.Compile(s)
+ if err != nil {
+ return invalidMatcher{}
+ }
+ return cs
+}
+
+// invalidMatcher is a Matcher that always fails to match.
+type invalidMatcher struct{}
+
+func (invalidMatcher) Match(n *html.Node) bool { return false }
+func (invalidMatcher) MatchAll(n *html.Node) []*html.Node { return nil }
+func (invalidMatcher) Filter(ns []*html.Node) []*html.Node { return nil }
diff --git a/vendor/github.com/PuerkitoBio/goquery/type_test.go b/vendor/github.com/PuerkitoBio/goquery/type_test.go
new file mode 100644
index 0000000..1e82d5e
--- /dev/null
+++ b/vendor/github.com/PuerkitoBio/goquery/type_test.go
@@ -0,0 +1,202 @@
+package goquery
+
+import (
+ "bytes"
+ "fmt"
+ "os"
+ "strings"
+ "testing"
+
+ "golang.org/x/net/html"
+)
+
+// Test helper functions and members
+var doc *Document
+var doc2 *Document
+var doc3 *Document
+var docB *Document
+var docW *Document
+
+func Doc() *Document {
+ if doc == nil {
+ doc = loadDoc("page.html")
+ }
+ return doc
+}
+
+func Doc2() *Document {
+ if doc2 == nil {
+ doc2 = loadDoc("page2.html")
+ }
+ return doc2
+}
+
+func Doc2Clone() *Document {
+ return CloneDocument(Doc2())
+}
+
+func Doc3() *Document {
+ if doc3 == nil {
+ doc3 = loadDoc("page3.html")
+ }
+ return doc3
+}
+
+func Doc3Clone() *Document {
+ return CloneDocument(Doc3())
+}
+
+func DocB() *Document {
+ if docB == nil {
+ docB = loadDoc("gotesting.html")
+ }
+ return docB
+}
+
+func DocW() *Document {
+ if docW == nil {
+ docW = loadDoc("gowiki.html")
+ }
+ return docW
+}
+
+func assertLength(t *testing.T, nodes []*html.Node, length int) {
+ if len(nodes) != length {
+ t.Errorf("Expected %d nodes, found %d.", length, len(nodes))
+ for i, n := range nodes {
+ t.Logf("Node %d: %+v.", i, n)
+ }
+ }
+}
+
+func assertClass(t *testing.T, sel *Selection, class string) {
+ if !sel.HasClass(class) {
+ t.Errorf("Expected node to have class %s, found %+v.", class, sel.Get(0))
+ }
+}
+
+func assertPanic(t *testing.T) {
+ if e := recover(); e == nil {
+ t.Error("Expected a panic.")
+ }
+}
+
+func assertEqual(t *testing.T, s1 *Selection, s2 *Selection) {
+ if s1 != s2 {
+ t.Error("Expected selection objects to be the same.")
+ }
+}
+
+func assertSelectionIs(t *testing.T, sel *Selection, is ...string) {
+ for i := 0; i < sel.Length(); i++ {
+ if !sel.Eq(i).Is(is[i]) {
+ t.Errorf("Expected node %d to be %s, found %+v", i, is[i], sel.Get(i))
+ }
+ }
+}
+
+func printSel(t *testing.T, sel *Selection) {
+ if testing.Verbose() {
+ h, err := sel.Html()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log(h)
+ }
+}
+
+func loadDoc(page string) *Document {
+ var f *os.File
+ var e error
+
+ if f, e = os.Open(fmt.Sprintf("./testdata/%s", page)); e != nil {
+ panic(e.Error())
+ }
+ defer f.Close()
+
+ var node *html.Node
+ if node, e = html.Parse(f); e != nil {
+ panic(e.Error())
+ }
+ return NewDocumentFromNode(node)
+}
+
+func TestNewDocument(t *testing.T) {
+ if f, e := os.Open("./testdata/page.html"); e != nil {
+ t.Error(e.Error())
+ } else {
+ defer f.Close()
+ if node, e := html.Parse(f); e != nil {
+ t.Error(e.Error())
+ } else {
+ doc = NewDocumentFromNode(node)
+ }
+ }
+}
+
+func TestNewDocumentFromReader(t *testing.T) {
+ cases := []struct {
+ src string
+ err bool
+ sel string
+ cnt int
+ }{
+ 0: {
+ src: `
+
+
++ This is some text. +
+ + + + +` + +func TestNodeName(t *testing.T) { + doc, err := NewDocumentFromReader(strings.NewReader(allNodes)) + if err != nil { + t.Fatal(err) + } + + n0 := doc.Nodes[0] + nDT := n0.FirstChild + sMeta := doc.Find("meta") + nMeta := sMeta.Get(0) + sP := doc.Find("p") + nP := sP.Get(0) + nComment := nP.FirstChild + nText := nComment.NextSibling + + cases := []struct { + node *html.Node + typ html.NodeType + want string + }{ + {n0, html.DocumentNode, nodeNames[html.DocumentNode]}, + {nDT, html.DoctypeNode, "html"}, + {nMeta, html.ElementNode, "meta"}, + {nP, html.ElementNode, "p"}, + {nComment, html.CommentNode, nodeNames[html.CommentNode]}, + {nText, html.TextNode, nodeNames[html.TextNode]}, + } + for i, c := range cases { + got := NodeName(newSingleSelection(c.node, doc)) + if c.node.Type != c.typ { + t.Errorf("%d: want type %v, got %v", i, c.typ, c.node.Type) + } + if got != c.want { + t.Errorf("%d: want %q, got %q", i, c.want, got) + } + } +} + +func TestNodeNameMultiSel(t *testing.T) { + doc, err := NewDocumentFromReader(strings.NewReader(allNodes)) + if err != nil { + t.Fatal(err) + } + + in := []string{"p", "h1", "div"} + var out []string + doc.Find(strings.Join(in, ", ")).Each(func(i int, s *Selection) { + got := NodeName(s) + out = append(out, got) + }) + sort.Strings(in) + sort.Strings(out) + if !reflect.DeepEqual(in, out) { + t.Errorf("want %v, got %v", in, out) + } +} + +func TestOuterHtml(t *testing.T) { + doc, err := NewDocumentFromReader(strings.NewReader(allNodes)) + if err != nil { + t.Fatal(err) + } + + n0 := doc.Nodes[0] + nDT := n0.FirstChild + sMeta := doc.Find("meta") + sP := doc.Find("p") + nP := sP.Get(0) + nComment := nP.FirstChild + nText := nComment.NextSibling + sHeaders := doc.Find(".header") + + cases := []struct { + node *html.Node + sel *Selection + want string + }{ + {nDT, nil, ""}, // render makes DOCTYPE all caps + {nil, sMeta, ``}, // and auto-closes the meta + {nil, sP, `+ This is some text. +
`}, + {nComment, nil, ""}, + {nText, nil, ` + This is some text. + `}, + {nil, sHeaders, ``}, + } + for i, c := range cases { + if c.sel == nil { + c.sel = newSingleSelection(c.node, doc) + } + got, err := OuterHtml(c.sel) + if err != nil { + t.Fatal(err) + } + + if got != c.want { + t.Errorf("%d: want %q, got %q", i, c.want, got) + } + } +} diff --git a/vendor/github.com/andybalholm/cascadia/LICENSE b/vendor/github.com/andybalholm/cascadia/LICENSE new file mode 100755 index 0000000..ee5ad35 --- /dev/null +++ b/vendor/github.com/andybalholm/cascadia/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2011 Andy Balholm. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/andybalholm/cascadia/README.md b/vendor/github.com/andybalholm/cascadia/README.md new file mode 100644 index 0000000..26f4c37 --- /dev/null +++ b/vendor/github.com/andybalholm/cascadia/README.md @@ -0,0 +1,9 @@ +# cascadia + +[](https://travis-ci.org/andybalholm/cascadia) + +The Cascadia package implements CSS selectors for use with the parse trees produced by the html package. + +To test CSS selectors without writing Go code, check out [cascadia](https://github.com/suntong/cascadia) the command line tool, a thin wrapper around this package. + +[Refer to godoc here](https://godoc.org/github.com/andybalholm/cascadia). diff --git a/vendor/github.com/andybalholm/cascadia/benchmark_test.go b/vendor/github.com/andybalholm/cascadia/benchmark_test.go new file mode 100644 index 0000000..42bf500 --- /dev/null +++ b/vendor/github.com/andybalholm/cascadia/benchmark_test.go @@ -0,0 +1,53 @@ +package cascadia + +import ( + "strings" + "testing" + + "golang.org/x/net/html" +) + +func MustParseHTML(doc string) *html.Node { + dom, err := html.Parse(strings.NewReader(doc)) + if err != nil { + panic(err) + } + return dom +} + +var selector = MustCompile(`div.matched`) +var doc = ` + + +`, + "#foo", + []string{ + `
`, + }, + }, + { + ``, + "li#t1", + []string{ + `
`, + "p.t1", + []string{ + `
`, + }, + }, + { + ``, + ".t1.fail", + []string{}, + }, + { + `
`, + "p.t1.t2", + []string{ + `
`, + }, + }, + { + ``, + "p[title]", + []string{ + `
`, + }, + }, + { + ``, + `address[title="foo"]`, + []string{ + ``, + }, + }, + { + ``, + `address[title!="foo"]`, + []string{ + ``, + ``, + }, + }, + { + ``, + `[ title ~= foo ]`, + []string{ + `
`, + }, + }, + { + ``, + `[title~="hello world"]`, + []string{}, + }, + { + `
`, + `[lang|="en"]`, + []string{ + `
`, + ``, + }, + }, + { + ``, + `[title^="foo"]`, + []string{ + `
`, + }, + }, + { + ``, + `[title$="bar"]`, + []string{ + `
`, + }, + }, + { + ``, + `[title*="bar"]`, + []string{ + `
`, + }, + }, + { + `This text should be green.
This text should be green.
`, + `p[class$=" "]`, + []string{}, + }, + { + `This text should be green.
This text should be green.
`, + `p[class$=""]`, + []string{}, + }, + { + `This text should be green.
This text should be green.
`, + `p[class^=" "]`, + []string{}, + }, + { + `This text should be green.
This text should be green.
`, + `p[class^=""]`, + []string{}, + }, + { + `This text should be green.
This text should be green.
`, + `p[class*=" "]`, + []string{}, + }, + { + `This text should be green.
This text should be green.
`, + `p[class*=""]`, + []string{}, + }, + { + ``, + `input[name=Sex][value=F]`, + []string{ + ``, + }, + }, + { + ``, + ".t1:not(.t2)", + []string{}, + }, + { + `
some text and a span and another
`, + `span:first-child`, + []string{ + `and a span`, + }, + }, + { + `a span and some text`, + `span:last-child`, + []string{ + `a span`, + }, + }, + { + ``, + `p:nth-of-type(2)`, + []string{ + `
`, + }, + }, + { + ``, + `p:nth-last-of-type(2)`, + []string{ + ``, + }, + }, + { + ``, + `p:last-of-type`, + []string{ + ``, + }, + }, + { + ``, + `p:first-of-type`, + []string{ + ``, + }, + }, + { + `Hello
`,
+ `:empty`,
+ []string{
+ ``,
+ ``,
+ ``,
+ },
+ },
+ {
+ ` `,
+ `div p`,
+ []string{
+ `
`, + `div table p`, + []string{ + `
`, + }, + }, + { + ``, + `p ~ p`, + []string{ + `
`, + ``, + }, + }, + { + ` + +`, + `p + p`, + []string{ + `
`, + }, + }, + { + ``, + `li, p`, + []string{ + "
", + "", + "", + }, + }, + { + ``, + `p +/*This is a comment*/ p`, + []string{ + `
`, + }, + }, + { + `Text block that wraps inner text and continues
`, + `p:contains("that wraps")`, + []string{ + `Text block that wraps inner text and continues
`, + }, + }, + { + `Text block that wraps inner text and continues
`, + `p:containsOwn("that wraps")`, + []string{}, + }, + { + `Text block that wraps inner text and continues
`, + `:containsOwn("inner")`, + []string{ + `wraps inner text`, + }, + }, + { + `Text block that wraps inner text and continues
`, + `p:containsOwn("block")`, + []string{ + `Text block that wraps inner text and continues
`, + }, + }, + { + `text content
text content
contents 1
contents 2
contents 2
contents 1
contents 2
contents 2
contents 2
`, + }, + }, + { + `contents 1
contents 2
contents 2
`, + }, + }, + { + `0123456789
abcdef
0123ABCD
`, + `p:matches([\d])`, + []string{ + `0123456789
`, + `0123ABCD
`, + }, + }, + { + `0123456789
abcdef
0123ABCD
`, + `p:matches([a-z])`, + []string{ + `abcdef
`, + }, + }, + { + `0123456789
abcdef
0123ABCD
`, + `p:matches([a-zA-Z])`, + []string{ + `abcdef
`, + `0123ABCD
`, + }, + }, + { + `0123456789
abcdef
0123ABCD
`, + `p:matches([^\d])`, + []string{ + `abcdef
`, + `0123ABCD
`, + }, + }, + { + `0123456789
abcdef
0123ABCD
`, + `p:matches(^(0|a))`, + []string{ + `0123456789
`, + `abcdef
`, + `0123ABCD
`, + }, + }, + { + `0123456789
abcdef
0123ABCD
`, + `p:matches(^\d+$)`, + []string{ + `0123456789
`, + }, + }, + { + `0123456789
abcdef
0123ABCD
`, + `p:not(:matches(^\d+$))`, + []string{ + `abcdef
`, + `0123ABCD
`, + }, + }, + { + `0123456789
0123456789
`, + `567`, + }, + }, + { + ``, + `[href#=(fina)]:not([href#=(\/\/[^\/]+untrusted)])`, + []string{ + ``, + ``, + }, + }, + { + ``, + `[href#=(^https:\/\/[^\/]*\/?news)]`, + []string{ + ``, + }, + }, + { + ``, + `:input`, + []string{ + ``, + ``, + ``, + ``, + ``, + }, + }, + { + ``, + ":root", + []string{ + "", + }, + }, + { + ``, + "*:root", + []string{ + "", + }, + }, + { + ``, + "*:root:first-child", + []string{}, + }, + { + ``, + "*:root:nth-child(1)", + []string{}, + }, + { + ``, + "a:not(:root)", + []string{ + ``, + }, + }, +} + +func TestSelectors(t *testing.T) { + for _, test := range selectorTests { + s, err := Compile(test.selector) + if err != nil { + t.Errorf("error compiling %q: %s", test.selector, err) + continue + } + + doc, err := html.Parse(strings.NewReader(test.HTML)) + if err != nil { + t.Errorf("error parsing %q: %s", test.HTML, err) + continue + } + + matches := s.MatchAll(doc) + if len(matches) != len(test.results) { + t.Errorf("selector %s wanted %d elements, got %d instead", test.selector, len(test.results), len(matches)) + continue + } + + for i, m := range matches { + got := nodeString(m) + if got != test.results[i] { + t.Errorf("selector %s wanted %s, got %s instead", test.selector, test.results[i], got) + } + } + + firstMatch := s.MatchFirst(doc) + if len(test.results) == 0 { + if firstMatch != nil { + t.Errorf("MatchFirst: selector %s want nil, got %s", test.selector, nodeString(firstMatch)) + } + } else { + got := nodeString(firstMatch) + if got != test.results[0] { + t.Errorf("MatchFirst: selector %s want %s, got %s", test.selector, test.results[0], got) + } + } + } +} diff --git a/vendor/github.com/antchfx/htmlquery/LICENSE b/vendor/github.com/antchfx/htmlquery/LICENSE new file mode 100644 index 0000000..e14c371 --- /dev/null +++ b/vendor/github.com/antchfx/htmlquery/LICENSE @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/antchfx/htmlquery/README.md b/vendor/github.com/antchfx/htmlquery/README.md new file mode 100644 index 0000000..3d16f55 --- /dev/null +++ b/vendor/github.com/antchfx/htmlquery/README.md @@ -0,0 +1,93 @@ +htmlquery +==== +[](https://travis-ci.org/antchfx/htmlquery) +[](https://coveralls.io/github/antchfx/htmlquery?branch=master) +[](https://godoc.org/github.com/antchfx/htmlquery) +[](https://goreportcard.com/report/github.com/antchfx/htmlquery) + +Overview +==== + +htmlquery is an XPath query package for HTML, lets you extract data or evaluate from HTML documents by an XPath expression. + +Installation +==== + +> $ go get github.com/antchfx/htmlquery + +Getting Started +==== + +#### Load HTML document from URL. + +```go +doc, err := htmlquery.LoadURL("http://example.com/") +``` + +#### Load HTML document from string. + +```go +s := `....` +doc, err := htmlquery.Parse(strings.NewReader(s)) +``` + +#### Find all A elements. + +```go +list := htmlquery.Find(doc, "//a") +``` + +#### Find all A elements that have `href` attribute. + +```go +list := range htmlquery.Find(doc, "//a[@href]") +``` + +#### Find all A elements and only get `href` attribute self. + +```go +list := range htmlquery.Find(doc, "//a/@href") +``` + +### Find the third A element. + +```go +a := htmlquery.FindOne(doc, "//a[3]") +``` + +#### Evaluate the number of all IMG element. + +```go +expr, _ := xpath.Compile("count(//img)") +v := expr.Evaluate(htmlquery.CreateXPathNavigator(doc)).(float64) +fmt.Printf("total count is %f", v) +``` + +Quick Tutorial +=== + +```go +func main() { + doc, err := htmlquery.LoadURL("https://www.bing.com/search?q=golang") + if err != nil { + panic(err) + } + // Find all news item. + for i, n := range htmlquery.Find(doc, "//ol/li") { + a := htmlquery.FindOne(n, "//a") + fmt.Printf("%d %s(%s)\n", i, htmlquery.InnerText(a), htmlquery.SelectAttr(a, "href")) + } +} +``` + +List of supported XPath query packages +=== +|Name |Description | +|--------------------------|----------------| +|[htmlquery](https://github.com/antchfx/htmlquery) | XPath query package for the HTML document| +|[xmlquery](https://github.com/antchfx/xmlquery) | XPath query package for the XML document| +|[jsonquery](https://github.com/antchfx/jsonquery) | XPath query package for the JSON document| + +Questions +=== +Please let me know if you have any questions. diff --git a/vendor/github.com/antchfx/htmlquery/query.go b/vendor/github.com/antchfx/htmlquery/query.go new file mode 100644 index 0000000..3b93256 --- /dev/null +++ b/vendor/github.com/antchfx/htmlquery/query.go @@ -0,0 +1,299 @@ +/* +Package htmlquery provides extract data from HTML documents using XPath expression. +*/ +package htmlquery + +import ( + "bytes" + "fmt" + "io" + "net/http" + + "github.com/antchfx/xpath" + "golang.org/x/net/html" + "golang.org/x/net/html/charset" +) + +var _ xpath.NodeNavigator = &NodeNavigator{} + +// CreateXPathNavigator creates a new xpath.NodeNavigator for the specified html.Node. +func CreateXPathNavigator(top *html.Node) *NodeNavigator { + return &NodeNavigator{curr: top, root: top, attr: -1} +} + +// Find searches the html.Node that matches by the specified XPath expr. +func Find(top *html.Node, expr string) []*html.Node { + var elems []*html.Node + exp, err := xpath.Compile(expr) + if err != nil { + panic(err) + } + t := exp.Select(CreateXPathNavigator(top)) + for t.MoveNext() { + elems = append(elems, getCurrentNode(t)) + } + return elems +} + +// FindOne searches the html.Node that matches by the specified XPath expr, +// and returns first element of matched html.Node. +func FindOne(top *html.Node, expr string) *html.Node { + var elem *html.Node + exp, err := xpath.Compile(expr) + if err != nil { + panic(err) + } + t := exp.Select(CreateXPathNavigator(top)) + if t.MoveNext() { + elem = getCurrentNode(t) + } + return elem +} + +// FindEach searches the html.Node and calls functions cb. +func FindEach(top *html.Node, expr string, cb func(int, *html.Node)) { + exp, err := xpath.Compile(expr) + if err != nil { + panic(err) + } + t := exp.Select(CreateXPathNavigator(top)) + i := 0 + for t.MoveNext() { + cb(i, getCurrentNode(t)) + i++ + } +} + +// LoadURL loads the HTML document from the specified URL. +func LoadURL(url string) (*html.Node, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + r, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type")) + if err != nil { + return nil, err + } + return html.Parse(r) +} + +func getCurrentNode(it *xpath.NodeIterator) *html.Node { + n := it.Current().(*NodeNavigator) + if n.NodeType() == xpath.AttributeNode { + childNode := &html.Node{ + Type: html.TextNode, + Data: n.Value(), + } + return &html.Node{ + Type: html.ElementNode, + Data: n.LocalName(), + FirstChild: childNode, + LastChild: childNode, + } + + } + return n.curr +} + +// Parse returns the parse tree for the HTML from the given Reader. +func Parse(r io.Reader) (*html.Node, error) { + return html.Parse(r) +} + +// InnerText returns the text between the start and end tags of the object. +func InnerText(n *html.Node) string { + var output func(*bytes.Buffer, *html.Node) + output = func(buf *bytes.Buffer, n *html.Node) { + switch n.Type { + case html.TextNode: + buf.WriteString(n.Data) + return + case html.CommentNode: + return + } + for child := n.FirstChild; child != nil; child = child.NextSibling { + output(buf, child) + } + } + + var buf bytes.Buffer + output(&buf, n) + return buf.String() +} + +// SelectAttr returns the attribute value with the specified name. +func SelectAttr(n *html.Node, name string) (val string) { + if n == nil { + return + } + if n.Type == html.ElementNode && n.Parent == nil && name == n.Data { + return InnerText(n) + } + for _, attr := range n.Attr { + if attr.Key == name { + val = attr.Val + break + } + } + return +} + +// OutputHTML returns the text including tags name. +func OutputHTML(n *html.Node, self bool) string { + var buf bytes.Buffer + if self { + html.Render(&buf, n) + } else { + for n := n.FirstChild; n != nil; n = n.NextSibling { + html.Render(&buf, n) + } + } + return buf.String() +} + +type NodeNavigator struct { + root, curr *html.Node + attr int +} + +func (h *NodeNavigator) Current() *html.Node { + return h.curr +} + +func (h *NodeNavigator) NodeType() xpath.NodeType { + switch h.curr.Type { + case html.CommentNode: + return xpath.CommentNode + case html.TextNode: + return xpath.TextNode + case html.DocumentNode: + return xpath.RootNode + case html.ElementNode: + if h.attr != -1 { + return xpath.AttributeNode + } + return xpath.ElementNode + case html.DoctypeNode: + // ignored declare and as Root-Node type. + return xpath.RootNode + } + panic(fmt.Sprintf("unknown HTML node type: %v", h.curr.Type)) +} + +func (h *NodeNavigator) LocalName() string { + if h.attr != -1 { + return h.curr.Attr[h.attr].Key + } + return h.curr.Data +} + +func (*NodeNavigator) Prefix() string { + return "" +} + +func (h *NodeNavigator) Value() string { + switch h.curr.Type { + case html.CommentNode: + return h.curr.Data + case html.ElementNode: + if h.attr != -1 { + return h.curr.Attr[h.attr].Val + } + return InnerText(h.curr) + case html.TextNode: + return h.curr.Data + } + return "" +} + +func (h *NodeNavigator) Copy() xpath.NodeNavigator { + n := *h + return &n +} + +func (h *NodeNavigator) MoveToRoot() { + h.curr = h.root +} + +func (h *NodeNavigator) MoveToParent() bool { + if h.attr != -1 { + h.attr = -1 + return true + } else if node := h.curr.Parent; node != nil { + h.curr = node + return true + } + return false +} + +func (h *NodeNavigator) MoveToNextAttribute() bool { + if h.attr >= len(h.curr.Attr)-1 { + return false + } + h.attr++ + return true +} + +func (h *NodeNavigator) MoveToChild() bool { + if h.attr != -1 { + return false + } + if node := h.curr.FirstChild; node != nil { + h.curr = node + return true + } + return false +} + +func (h *NodeNavigator) MoveToFirst() bool { + if h.attr != -1 || h.curr.PrevSibling == nil { + return false + } + for { + node := h.curr.PrevSibling + if node == nil { + break + } + h.curr = node + } + return true +} + +func (h *NodeNavigator) String() string { + return h.Value() +} + +func (h *NodeNavigator) MoveToNext() bool { + if h.attr != -1 { + return false + } + if node := h.curr.NextSibling; node != nil { + h.curr = node + return true + } + return false +} + +func (h *NodeNavigator) MoveToPrevious() bool { + if h.attr != -1 { + return false + } + if node := h.curr.PrevSibling; node != nil { + h.curr = node + return true + } + return false +} + +func (h *NodeNavigator) MoveTo(other xpath.NodeNavigator) bool { + node, ok := other.(*NodeNavigator) + if !ok || node.root != h.root { + return false + } + + h.curr = node.curr + h.attr = node.attr + return true +} diff --git a/vendor/github.com/antchfx/htmlquery/query_test.go b/vendor/github.com/antchfx/htmlquery/query_test.go new file mode 100644 index 0000000..d3503b7 --- /dev/null +++ b/vendor/github.com/antchfx/htmlquery/query_test.go @@ -0,0 +1,126 @@ +package htmlquery + +import ( + "strings" + "testing" + + "github.com/antchfx/xpath" + "golang.org/x/net/html" +) + +const htmlSample = ` + +
+ London is the capital city of England. It is the most populous city in the United Kingdom, with a metropolitan area of over 13 million inhabitants.
+Standing on the River Thames, London has been a major settlement for two millennia, its history going back to its founding by the Romans, who named it Londinium.
++ Hello,This is an example for gxpath. +
+ + + + */ + doc := createNode("", RootNode) + xhtml := doc.createChildNode("html", ElementNode) + xhtml.addAttribute("lang", "en") + + // The HTML head section. + head := xhtml.createChildNode("head", ElementNode) + n := head.createChildNode("title", ElementNode) + n = n.createChildNode("Hello", TextNode) + n = head.createChildNode("meta", ElementNode) + n.addAttribute("name", "language") + n.addAttribute("content", "en") + // The HTML body section. + body := xhtml.createChildNode("body", ElementNode) + n = body.createChildNode("h1", ElementNode) + n = n.createChildNode(" This is a H1 ", TextNode) + ul := body.createChildNode("ul", ElementNode) + n = ul.createChildNode("li", ElementNode) + n = n.createChildNode("a", ElementNode) + n.addAttribute("id", "1") + n.addAttribute("href", "/") + n = n.createChildNode("Home", TextNode) + n = ul.createChildNode("li", ElementNode) + n = n.createChildNode("a", ElementNode) + n.addAttribute("id", "2") + n.addAttribute("href", "/about") + n = n.createChildNode("about", TextNode) + n = ul.createChildNode("li", ElementNode) + n = n.createChildNode("a", ElementNode) + n.addAttribute("id", "3") + n.addAttribute("href", "/account") + n = n.createChildNode("login", TextNode) + n = ul.createChildNode("li", ElementNode) + + n = body.createChildNode("p", ElementNode) + n = n.createChildNode("Hello,This is an example for gxpath.", TextNode) + + n = body.createChildNode("footer", ElementNode) + n = n.createChildNode("footer script", TextNode) + + return xhtml +} diff --git a/vendor/github.com/gobwas/glob/LICENSE b/vendor/github.com/gobwas/glob/LICENSE new file mode 100644 index 0000000..9d4735c --- /dev/null +++ b/vendor/github.com/gobwas/glob/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Sergey Kamardin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/gobwas/glob/bench.sh b/vendor/github.com/gobwas/glob/bench.sh new file mode 100755 index 0000000..804cf22 --- /dev/null +++ b/vendor/github.com/gobwas/glob/bench.sh @@ -0,0 +1,26 @@ +#! /bin/bash + +bench() { + filename="/tmp/$1-$2.bench" + if test -e "${filename}"; + then + echo "Already exists ${filename}" + else + backup=`git rev-parse --abbrev-ref HEAD` + git checkout $1 + echo -n "Creating ${filename}... " + go test ./... -run=NONE -bench=$2 > "${filename}" -benchmem + echo "OK" + git checkout ${backup} + sleep 5 + fi +} + + +to=$1 +current=`git rev-parse --abbrev-ref HEAD` + +bench ${to} $2 +bench ${current} $2 + +benchcmp $3 "/tmp/${to}-$2.bench" "/tmp/${current}-$2.bench" diff --git a/vendor/github.com/gobwas/glob/compiler/compiler.go b/vendor/github.com/gobwas/glob/compiler/compiler.go new file mode 100644 index 0000000..02e7de8 --- /dev/null +++ b/vendor/github.com/gobwas/glob/compiler/compiler.go @@ -0,0 +1,525 @@ +package compiler + +// TODO use constructor with all matchers, and to their structs private +// TODO glue multiple Text nodes (like after QuoteMeta) + +import ( + "fmt" + "reflect" + + "github.com/gobwas/glob/match" + "github.com/gobwas/glob/syntax/ast" + "github.com/gobwas/glob/util/runes" +) + +func optimizeMatcher(matcher match.Matcher) match.Matcher { + switch m := matcher.(type) { + + case match.Any: + if len(m.Separators) == 0 { + return match.NewSuper() + } + + case match.AnyOf: + if len(m.Matchers) == 1 { + return m.Matchers[0] + } + + return m + + case match.List: + if m.Not == false && len(m.List) == 1 { + return match.NewText(string(m.List)) + } + + return m + + case match.BTree: + m.Left = optimizeMatcher(m.Left) + m.Right = optimizeMatcher(m.Right) + + r, ok := m.Value.(match.Text) + if !ok { + return m + } + + var ( + leftNil = m.Left == nil + rightNil = m.Right == nil + ) + if leftNil && rightNil { + return match.NewText(r.Str) + } + + _, leftSuper := m.Left.(match.Super) + lp, leftPrefix := m.Left.(match.Prefix) + la, leftAny := m.Left.(match.Any) + + _, rightSuper := m.Right.(match.Super) + rs, rightSuffix := m.Right.(match.Suffix) + ra, rightAny := m.Right.(match.Any) + + switch { + case leftSuper && rightSuper: + return match.NewContains(r.Str, false) + + case leftSuper && rightNil: + return match.NewSuffix(r.Str) + + case rightSuper && leftNil: + return match.NewPrefix(r.Str) + + case leftNil && rightSuffix: + return match.NewPrefixSuffix(r.Str, rs.Suffix) + + case rightNil && leftPrefix: + return match.NewPrefixSuffix(lp.Prefix, r.Str) + + case rightNil && leftAny: + return match.NewSuffixAny(r.Str, la.Separators) + + case leftNil && rightAny: + return match.NewPrefixAny(r.Str, ra.Separators) + } + + return m + } + + return matcher +} + +func compileMatchers(matchers []match.Matcher) (match.Matcher, error) { + if len(matchers) == 0 { + return nil, fmt.Errorf("compile error: need at least one matcher") + } + if len(matchers) == 1 { + return matchers[0], nil + } + if m := glueMatchers(matchers); m != nil { + return m, nil + } + + idx := -1 + maxLen := -1 + var val match.Matcher + for i, matcher := range matchers { + if l := matcher.Len(); l != -1 && l >= maxLen { + maxLen = l + idx = i + val = matcher + } + } + + if val == nil { // not found matcher with static length + r, err := compileMatchers(matchers[1:]) + if err != nil { + return nil, err + } + return match.NewBTree(matchers[0], nil, r), nil + } + + left := matchers[:idx] + var right []match.Matcher + if len(matchers) > idx+1 { + right = matchers[idx+1:] + } + + var l, r match.Matcher + var err error + if len(left) > 0 { + l, err = compileMatchers(left) + if err != nil { + return nil, err + } + } + + if len(right) > 0 { + r, err = compileMatchers(right) + if err != nil { + return nil, err + } + } + + return match.NewBTree(val, l, r), nil +} + +func glueMatchers(matchers []match.Matcher) match.Matcher { + if m := glueMatchersAsEvery(matchers); m != nil { + return m + } + if m := glueMatchersAsRow(matchers); m != nil { + return m + } + return nil +} + +func glueMatchersAsRow(matchers []match.Matcher) match.Matcher { + if len(matchers) <= 1 { + return nil + } + + var ( + c []match.Matcher + l int + ) + for _, matcher := range matchers { + if ml := matcher.Len(); ml == -1 { + return nil + } else { + c = append(c, matcher) + l += ml + } + } + return match.NewRow(l, c...) +} + +func glueMatchersAsEvery(matchers []match.Matcher) match.Matcher { + if len(matchers) <= 1 { + return nil + } + + var ( + hasAny bool + hasSuper bool + hasSingle bool + min int + separator []rune + ) + + for i, matcher := range matchers { + var sep []rune + + switch m := matcher.(type) { + case match.Super: + sep = []rune{} + hasSuper = true + + case match.Any: + sep = m.Separators + hasAny = true + + case match.Single: + sep = m.Separators + hasSingle = true + min++ + + case match.List: + if !m.Not { + return nil + } + sep = m.List + hasSingle = true + min++ + + default: + return nil + } + + // initialize + if i == 0 { + separator = sep + } + + if runes.Equal(sep, separator) { + continue + } + + return nil + } + + if hasSuper && !hasAny && !hasSingle { + return match.NewSuper() + } + + if hasAny && !hasSuper && !hasSingle { + return match.NewAny(separator) + } + + if (hasAny || hasSuper) && min > 0 && len(separator) == 0 { + return match.NewMin(min) + } + + every := match.NewEveryOf() + + if min > 0 { + every.Add(match.NewMin(min)) + + if !hasAny && !hasSuper { + every.Add(match.NewMax(min)) + } + } + + if len(separator) > 0 { + every.Add(match.NewContains(string(separator), true)) + } + + return every +} + +func minimizeMatchers(matchers []match.Matcher) []match.Matcher { + var done match.Matcher + var left, right, count int + + for l := 0; l < len(matchers); l++ { + for r := len(matchers); r > l; r-- { + if glued := glueMatchers(matchers[l:r]); glued != nil { + var swap bool + + if done == nil { + swap = true + } else { + cl, gl := done.Len(), glued.Len() + swap = cl > -1 && gl > -1 && gl > cl + swap = swap || count < r-l + } + + if swap { + done = glued + left = l + right = r + count = r - l + } + } + } + } + + if done == nil { + return matchers + } + + next := append(append([]match.Matcher{}, matchers[:left]...), done) + if right < len(matchers) { + next = append(next, matchers[right:]...) + } + + if len(next) == len(matchers) { + return next + } + + return minimizeMatchers(next) +} + +// minimizeAnyOf tries to apply some heuristics to minimize number of nodes in given tree +func minimizeTree(tree *ast.Node) *ast.Node { + switch tree.Kind { + case ast.KindAnyOf: + return minimizeTreeAnyOf(tree) + default: + return nil + } +} + +// minimizeAnyOf tries to find common children of given node of AnyOf pattern +// it searches for common children from left and from right +// if any common children are found – then it returns new optimized ast tree +// else it returns nil +func minimizeTreeAnyOf(tree *ast.Node) *ast.Node { + if !areOfSameKind(tree.Children, ast.KindPattern) { + return nil + } + + commonLeft, commonRight := commonChildren(tree.Children) + commonLeftCount, commonRightCount := len(commonLeft), len(commonRight) + if commonLeftCount == 0 && commonRightCount == 0 { // there are no common parts + return nil + } + + var result []*ast.Node + if commonLeftCount > 0 { + result = append(result, ast.NewNode(ast.KindPattern, nil, commonLeft...)) + } + + var anyOf []*ast.Node + for _, child := range tree.Children { + reuse := child.Children[commonLeftCount : len(child.Children)-commonRightCount] + var node *ast.Node + if len(reuse) == 0 { + // this pattern is completely reduced by commonLeft and commonRight patterns + // so it become nothing + node = ast.NewNode(ast.KindNothing, nil) + } else { + node = ast.NewNode(ast.KindPattern, nil, reuse...) + } + anyOf = appendIfUnique(anyOf, node) + } + switch { + case len(anyOf) == 1 && anyOf[0].Kind != ast.KindNothing: + result = append(result, anyOf[0]) + case len(anyOf) > 1: + result = append(result, ast.NewNode(ast.KindAnyOf, nil, anyOf...)) + } + + if commonRightCount > 0 { + result = append(result, ast.NewNode(ast.KindPattern, nil, commonRight...)) + } + + return ast.NewNode(ast.KindPattern, nil, result...) +} + +func commonChildren(nodes []*ast.Node) (commonLeft, commonRight []*ast.Node) { + if len(nodes) <= 1 { + return + } + + // find node that has least number of children + idx := leastChildren(nodes) + if idx == -1 { + return + } + tree := nodes[idx] + treeLength := len(tree.Children) + + // allocate max able size for rightCommon slice + // to get ability insert elements in reverse order (from end to start) + // without sorting + commonRight = make([]*ast.Node, treeLength) + lastRight := treeLength // will use this to get results as commonRight[lastRight:] + + var ( + breakLeft bool + breakRight bool + commonTotal int + ) + for i, j := 0, treeLength-1; commonTotal < treeLength && j >= 0 && !(breakLeft && breakRight); i, j = i+1, j-1 { + treeLeft := tree.Children[i] + treeRight := tree.Children[j] + + for k := 0; k < len(nodes) && !(breakLeft && breakRight); k++ { + // skip least children node + if k == idx { + continue + } + + restLeft := nodes[k].Children[i] + restRight := nodes[k].Children[j+len(nodes[k].Children)-treeLength] + + breakLeft = breakLeft || !treeLeft.Equal(restLeft) + + // disable searching for right common parts, if left part is already overlapping + breakRight = breakRight || (!breakLeft && j <= i) + breakRight = breakRight || !treeRight.Equal(restRight) + } + + if !breakLeft { + commonTotal++ + commonLeft = append(commonLeft, treeLeft) + } + if !breakRight { + commonTotal++ + lastRight = j + commonRight[j] = treeRight + } + } + + commonRight = commonRight[lastRight:] + + return +} + +func appendIfUnique(target []*ast.Node, val *ast.Node) []*ast.Node { + for _, n := range target { + if reflect.DeepEqual(n, val) { + return target + } + } + return append(target, val) +} + +func areOfSameKind(nodes []*ast.Node, kind ast.Kind) bool { + for _, n := range nodes { + if n.Kind != kind { + return false + } + } + return true +} + +func leastChildren(nodes []*ast.Node) int { + min := -1 + idx := -1 + for i, n := range nodes { + if idx == -1 || (len(n.Children) < min) { + min = len(n.Children) + idx = i + } + } + return idx +} + +func compileTreeChildren(tree *ast.Node, sep []rune) ([]match.Matcher, error) { + var matchers []match.Matcher + for _, desc := range tree.Children { + m, err := compile(desc, sep) + if err != nil { + return nil, err + } + matchers = append(matchers, optimizeMatcher(m)) + } + return matchers, nil +} + +func compile(tree *ast.Node, sep []rune) (m match.Matcher, err error) { + switch tree.Kind { + case ast.KindAnyOf: + // todo this could be faster on pattern_alternatives_combine_lite (see glob_test.go) + if n := minimizeTree(tree); n != nil { + return compile(n, sep) + } + matchers, err := compileTreeChildren(tree, sep) + if err != nil { + return nil, err + } + return match.NewAnyOf(matchers...), nil + + case ast.KindPattern: + if len(tree.Children) == 0 { + return match.NewNothing(), nil + } + matchers, err := compileTreeChildren(tree, sep) + if err != nil { + return nil, err + } + m, err = compileMatchers(minimizeMatchers(matchers)) + if err != nil { + return nil, err + } + + case ast.KindAny: + m = match.NewAny(sep) + + case ast.KindSuper: + m = match.NewSuper() + + case ast.KindSingle: + m = match.NewSingle(sep) + + case ast.KindNothing: + m = match.NewNothing() + + case ast.KindList: + l := tree.Value.(ast.List) + m = match.NewList([]rune(l.Chars), l.Not) + + case ast.KindRange: + r := tree.Value.(ast.Range) + m = match.NewRange(r.Lo, r.Hi, r.Not) + + case ast.KindText: + t := tree.Value.(ast.Text) + m = match.NewText(t.Text) + + default: + return nil, fmt.Errorf("could not compile tree: unknown node type") + } + + return optimizeMatcher(m), nil +} + +func Compile(tree *ast.Node, sep []rune) (match.Matcher, error) { + m, err := compile(tree, sep) + if err != nil { + return nil, err + } + + return m, nil +} diff --git a/vendor/github.com/gobwas/glob/compiler/compiler_test.go b/vendor/github.com/gobwas/glob/compiler/compiler_test.go new file mode 100644 index 0000000..b58b1eb --- /dev/null +++ b/vendor/github.com/gobwas/glob/compiler/compiler_test.go @@ -0,0 +1,624 @@ +package compiler + +import ( + "github.com/gobwas/glob/match" + "github.com/gobwas/glob/match/debug" + "github.com/gobwas/glob/syntax/ast" + "reflect" + "testing" +) + +var separators = []rune{'.'} + +func TestCommonChildren(t *testing.T) { + for i, test := range []struct { + nodes []*ast.Node + left []*ast.Node + right []*ast.Node + }{ + { + nodes: []*ast.Node{ + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"z"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ), + }, + }, + { + nodes: []*ast.Node{ + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"z"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ), + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ), + }, + left: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"a"}), + }, + right: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"c"}), + }, + }, + { + nodes: []*ast.Node{ + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ast.NewNode(ast.KindText, ast.Text{"d"}), + ), + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ast.NewNode(ast.KindText, ast.Text{"d"}), + ), + }, + left: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + }, + right: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"c"}), + ast.NewNode(ast.KindText, ast.Text{"d"}), + }, + }, + { + nodes: []*ast.Node{ + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ), + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ), + }, + left: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + }, + right: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"c"}), + }, + }, + { + nodes: []*ast.Node{ + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"d"}), + ), + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"d"}), + ), + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"e"}), + ), + }, + left: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"a"}), + }, + right: []*ast.Node{}, + }, + } { + left, right := commonChildren(test.nodes) + if !nodesEqual(left, test.left) { + t.Errorf("[%d] left, right := commonChildren(); left = %v; want %v", i, left, test.left) + } + if !nodesEqual(right, test.right) { + t.Errorf("[%d] left, right := commonChildren(); right = %v; want %v", i, right, test.right) + } + } +} + +func nodesEqual(a, b []*ast.Node) bool { + if len(a) != len(b) { + return false + } + for i, av := range a { + if !av.Equal(b[i]) { + return false + } + } + return true +} + +func TestGlueMatchers(t *testing.T) { + for id, test := range []struct { + in []match.Matcher + exp match.Matcher + }{ + { + []match.Matcher{ + match.NewSuper(), + match.NewSingle(nil), + }, + match.NewMin(1), + }, + { + []match.Matcher{ + match.NewAny(separators), + match.NewSingle(separators), + }, + match.EveryOf{match.Matchers{ + match.NewMin(1), + match.NewContains(string(separators), true), + }}, + }, + { + []match.Matcher{ + match.NewSingle(nil), + match.NewSingle(nil), + match.NewSingle(nil), + }, + match.EveryOf{match.Matchers{ + match.NewMin(3), + match.NewMax(3), + }}, + }, + { + []match.Matcher{ + match.NewList([]rune{'a'}, true), + match.NewAny([]rune{'a'}), + }, + match.EveryOf{match.Matchers{ + match.NewMin(1), + match.NewContains("a", true), + }}, + }, + } { + act, err := compileMatchers(test.in) + if err != nil { + t.Errorf("#%d convert matchers error: %s", id, err) + continue + } + + if !reflect.DeepEqual(act, test.exp) { + t.Errorf("#%d unexpected convert matchers result:\nact: %#v;\nexp: %#v", id, act, test.exp) + continue + } + } +} + +func TestCompileMatchers(t *testing.T) { + for id, test := range []struct { + in []match.Matcher + exp match.Matcher + }{ + { + []match.Matcher{ + match.NewSuper(), + match.NewSingle(separators), + match.NewText("c"), + }, + match.NewBTree( + match.NewText("c"), + match.NewBTree( + match.NewSingle(separators), + match.NewSuper(), + nil, + ), + nil, + ), + }, + { + []match.Matcher{ + match.NewAny(nil), + match.NewText("c"), + match.NewAny(nil), + }, + match.NewBTree( + match.NewText("c"), + match.NewAny(nil), + match.NewAny(nil), + ), + }, + { + []match.Matcher{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + }, + match.NewRow( + 4, + match.Matchers{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + }..., + ), + }, + } { + act, err := compileMatchers(test.in) + if err != nil { + t.Errorf("#%d convert matchers error: %s", id, err) + continue + } + + if !reflect.DeepEqual(act, test.exp) { + t.Errorf("#%d unexpected convert matchers result:\nact: %#v\nexp: %#v", id, act, test.exp) + continue + } + } +} + +func TestConvertMatchers(t *testing.T) { + for id, test := range []struct { + in, exp []match.Matcher + }{ + { + []match.Matcher{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + match.NewAny(nil), + }, + []match.Matcher{ + match.NewRow( + 4, + []match.Matcher{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + }..., + ), + match.NewAny(nil), + }, + }, + { + []match.Matcher{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + match.NewAny(nil), + match.NewSingle(nil), + match.NewSingle(nil), + match.NewAny(nil), + }, + []match.Matcher{ + match.NewRow( + 3, + match.Matchers{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + }..., + ), + match.NewMin(3), + }, + }, + } { + act := minimizeMatchers(test.in) + if !reflect.DeepEqual(act, test.exp) { + t.Errorf("#%d unexpected convert matchers 2 result:\nact: %#v\nexp: %#v", id, act, test.exp) + continue + } + } +} + +func TestCompiler(t *testing.T) { + for id, test := range []struct { + ast *ast.Node + result match.Matcher + sep []rune + }{ + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ), + result: match.NewText("abc"), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ), + sep: separators, + result: match.NewAny(separators), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ), + result: match.NewSuper(), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindSuper, nil), + ), + result: match.NewSuper(), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindSingle, nil), + ), + sep: separators, + result: match.NewSingle(separators), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindRange, ast.Range{ + Lo: 'a', + Hi: 'z', + Not: true, + }), + ), + result: match.NewRange('a', 'z', true), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindList, ast.List{ + Chars: "abc", + Not: true, + }), + ), + result: match.NewList([]rune{'a', 'b', 'c'}, true), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindSingle, nil), + ast.NewNode(ast.KindSingle, nil), + ast.NewNode(ast.KindSingle, nil), + ), + sep: separators, + result: match.EveryOf{Matchers: match.Matchers{ + match.NewMin(3), + match.NewContains(string(separators), true), + }}, + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindSingle, nil), + ast.NewNode(ast.KindSingle, nil), + ast.NewNode(ast.KindSingle, nil), + ), + result: match.NewMin(3), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindSingle, nil), + ), + sep: separators, + result: match.NewBTree( + match.NewRow( + 4, + match.Matchers{ + match.NewText("abc"), + match.NewSingle(separators), + }..., + ), + match.NewAny(separators), + nil, + ), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"/"}), + ast.NewNode(ast.KindAnyOf, nil, + ast.NewNode(ast.KindText, ast.Text{"z"}), + ast.NewNode(ast.KindText, ast.Text{"ab"}), + ), + ast.NewNode(ast.KindSuper, nil), + ), + sep: separators, + result: match.NewBTree( + match.NewText("/"), + nil, + match.NewBTree( + match.NewAnyOf(match.NewText("z"), match.NewText("ab")), + nil, + match.NewSuper(), + ), + ), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindSuper, nil), + ast.NewNode(ast.KindSingle, nil), + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindSingle, nil), + ), + sep: separators, + result: match.NewBTree( + match.NewRow( + 5, + match.Matchers{ + match.NewSingle(separators), + match.NewText("abc"), + match.NewSingle(separators), + }..., + ), + match.NewSuper(), + nil, + ), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ), + result: match.NewSuffix("abc"), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindAny, nil), + ), + result: match.NewPrefix("abc"), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindText, ast.Text{"def"}), + ), + result: match.NewPrefixSuffix("abc", "def"), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindAny, nil), + ), + result: match.NewContains("abc", false), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindAny, nil), + ast.NewNode(ast.KindAny, nil), + ), + sep: separators, + result: match.NewBTree( + match.NewText("abc"), + match.NewAny(separators), + match.NewAny(separators), + ), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindSuper, nil), + ast.NewNode(ast.KindSingle, nil), + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindSuper, nil), + ast.NewNode(ast.KindSingle, nil), + ), + result: match.NewBTree( + match.NewText("abc"), + match.NewMin(1), + match.NewMin(1), + ), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ), + result: match.NewText("abc"), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAnyOf, nil, + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAnyOf, nil, + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ), + ), + ), + ), + ), + result: match.NewText("abc"), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAnyOf, nil, + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindSingle, nil), + ), + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindList, ast.List{Chars: "def"}), + ), + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ), + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ), + ), + ), + result: match.NewBTree( + match.NewText("abc"), + nil, + match.AnyOf{Matchers: match.Matchers{ + match.NewSingle(nil), + match.NewList([]rune{'d', 'e', 'f'}, false), + match.NewNothing(), + }}, + ), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindRange, ast.Range{Lo: 'a', Hi: 'z'}), + ast.NewNode(ast.KindRange, ast.Range{Lo: 'a', Hi: 'x', Not: true}), + ast.NewNode(ast.KindAny, nil), + ), + result: match.NewBTree( + match.NewRow( + 2, + match.Matchers{ + match.NewRange('a', 'z', false), + match.NewRange('a', 'x', true), + }..., + ), + nil, + match.NewSuper(), + ), + }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAnyOf, nil, + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindList, ast.List{Chars: "abc"}), + ast.NewNode(ast.KindText, ast.Text{"ghi"}), + ), + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, ast.Text{"abc"}), + ast.NewNode(ast.KindList, ast.List{Chars: "def"}), + ast.NewNode(ast.KindText, ast.Text{"ghi"}), + ), + ), + ), + result: match.NewRow( + 7, + match.Matchers{ + match.NewText("abc"), + match.AnyOf{Matchers: match.Matchers{ + match.NewList([]rune{'a', 'b', 'c'}, false), + match.NewList([]rune{'d', 'e', 'f'}, false), + }}, + match.NewText("ghi"), + }..., + ), + }, + } { + m, err := Compile(test.ast, test.sep) + if err != nil { + t.Errorf("compilation error: %s", err) + continue + } + + if !reflect.DeepEqual(m, test.result) { + t.Errorf("[%d] Compile():\nexp: %#v\nact: %#v\n\ngraphviz:\nexp:\n%s\nact:\n%s\n", id, test.result, m, debug.Graphviz("", test.result.(match.Matcher)), debug.Graphviz("", m.(match.Matcher))) + continue + } + } +} diff --git a/vendor/github.com/gobwas/glob/glob.go b/vendor/github.com/gobwas/glob/glob.go new file mode 100644 index 0000000..2afde34 --- /dev/null +++ b/vendor/github.com/gobwas/glob/glob.go @@ -0,0 +1,80 @@ +package glob + +import ( + "github.com/gobwas/glob/compiler" + "github.com/gobwas/glob/syntax" +) + +// Glob represents compiled glob pattern. +type Glob interface { + Match(string) bool +} + +// Compile creates Glob for given pattern and strings (if any present after pattern) as separators. +// The pattern syntax is: +// +// pattern: +// { term } +// +// term: +// `*` matches any sequence of non-separator characters +// `**` matches any sequence of characters +// `?` matches any single non-separator character +// `[` [ `!` ] { character-range } `]` +// character class (must be non-empty) +// `{` pattern-list `}` +// pattern alternatives +// c matches character c (c != `*`, `**`, `?`, `\`, `[`, `{`, `}`) +// `\` c matches character c +// +// character-range: +// c matches character c (c != `\\`, `-`, `]`) +// `\` c matches character c +// lo `-` hi matches character c for lo <= c <= hi +// +// pattern-list: +// pattern { `,` pattern } +// comma-separated (without spaces) patterns +// +func Compile(pattern string, separators ...rune) (Glob, error) { + ast, err := syntax.Parse(pattern) + if err != nil { + return nil, err + } + + matcher, err := compiler.Compile(ast, separators) + if err != nil { + return nil, err + } + + return matcher, nil +} + +// MustCompile is the same as Compile, except that if Compile returns error, this will panic +func MustCompile(pattern string, separators ...rune) Glob { + g, err := Compile(pattern, separators...) + if err != nil { + panic(err) + } + + return g +} + +// QuoteMeta returns a string that quotes all glob pattern meta characters +// inside the argument text; For example, QuoteMeta(`{foo*}`) returns `\[foo\*\]`. +func QuoteMeta(s string) string { + b := make([]byte, 2*len(s)) + + // a byte loop is correct because all meta characters are ASCII + j := 0 + for i := 0; i < len(s); i++ { + if syntax.Special(s[i]) { + b[j] = '\\' + j++ + } + b[j] = s[i] + j++ + } + + return string(b[0:j]) +} diff --git a/vendor/github.com/gobwas/glob/glob_test.go b/vendor/github.com/gobwas/glob/glob_test.go new file mode 100644 index 0000000..810036f --- /dev/null +++ b/vendor/github.com/gobwas/glob/glob_test.go @@ -0,0 +1,531 @@ +package glob + +import ( + "regexp" + "testing" +) + +const ( + pattern_all = "[a-z][!a-x]*cat*[h][!b]*eyes*" + regexp_all = `^[a-z][^a-x].*cat.*[h][^b].*eyes.*$` + fixture_all_match = "my cat has very bright eyes" + fixture_all_mismatch = "my dog has very bright eyes" + + pattern_plain = "google.com" + regexp_plain = `^google\.com$` + fixture_plain_match = "google.com" + fixture_plain_mismatch = "gobwas.com" + + pattern_multiple = "https://*.google.*" + regexp_multiple = `^https:\/\/.*\.google\..*$` + fixture_multiple_match = "https://account.google.com" + fixture_multiple_mismatch = "https://google.com" + + pattern_alternatives = "{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}" + regexp_alternatives = `^(https:\/\/.*\.google\..*|.*yandex\..*|.*yahoo\..*|.*mail\.ru)$` + fixture_alternatives_match = "http://yahoo.com" + fixture_alternatives_mismatch = "http://google.com" + + pattern_alternatives_suffix = "{https://*gobwas.com,http://exclude.gobwas.com}" + regexp_alternatives_suffix = `^(https:\/\/.*gobwas\.com|http://exclude.gobwas.com)$` + fixture_alternatives_suffix_first_match = "https://safe.gobwas.com" + fixture_alternatives_suffix_first_mismatch = "http://safe.gobwas.com" + fixture_alternatives_suffix_second = "http://exclude.gobwas.com" + + pattern_prefix = "abc*" + regexp_prefix = `^abc.*$` + pattern_suffix = "*def" + regexp_suffix = `^.*def$` + pattern_prefix_suffix = "ab*ef" + regexp_prefix_suffix = `^ab.*ef$` + fixture_prefix_suffix_match = "abcdef" + fixture_prefix_suffix_mismatch = "af" + + pattern_alternatives_combine_lite = "{abc*def,abc?def,abc[zte]def}" + regexp_alternatives_combine_lite = `^(abc.*def|abc.def|abc[zte]def)$` + fixture_alternatives_combine_lite = "abczdef" + + pattern_alternatives_combine_hard = "{abc*[a-c]def,abc?[d-g]def,abc[zte]?def}" + regexp_alternatives_combine_hard = `^(abc.*[a-c]def|abc.[d-g]def|abc[zte].def)$` + fixture_alternatives_combine_hard = "abczqdef" +) + +type test struct { + pattern, match string + should bool + delimiters []rune +} + +func glob(s bool, p, m string, d ...rune) test { + return test{p, m, s, d} +} + +func TestGlob(t *testing.T) { + for _, test := range []test{ + glob(true, "* ?at * eyes", "my cat has very bright eyes"), + + glob(true, "", ""), + glob(false, "", "b"), + + glob(true, "*ä", "åä"), + glob(true, "abc", "abc"), + glob(true, "a*c", "abc"), + glob(true, "a*c", "a12345c"), + glob(true, "a?c", "a1c"), + glob(true, "a.b", "a.b", '.'), + glob(true, "a.*", "a.b", '.'), + glob(true, "a.**", "a.b.c", '.'), + glob(true, "a.?.c", "a.b.c", '.'), + glob(true, "a.?.?", "a.b.c", '.'), + glob(true, "?at", "cat"), + glob(true, "?at", "fat"), + glob(true, "*", "abc"), + glob(true, `\*`, "*"), + glob(true, "**", "a.b.c", '.'), + + glob(false, "?at", "at"), + glob(false, "?at", "fat", 'f'), + glob(false, "a.*", "a.b.c", '.'), + glob(false, "a.?.c", "a.bb.c", '.'), + glob(false, "*", "a.b.c", '.'), + + glob(true, "*test", "this is a test"), + glob(true, "this*", "this is a test"), + glob(true, "*is *", "this is a test"), + glob(true, "*is*a*", "this is a test"), + glob(true, "**test**", "this is a test"), + glob(true, "**is**a***test*", "this is a test"), + + glob(false, "*is", "this is a test"), + glob(false, "*no*", "this is a test"), + glob(true, "[!a]*", "this is a test3"), + + glob(true, "*abc", "abcabc"), + glob(true, "**abc", "abcabc"), + glob(true, "???", "abc"), + glob(true, "?*?", "abc"), + glob(true, "?*?", "ac"), + glob(false, "sta", "stagnation"), + glob(true, "sta*", "stagnation"), + glob(false, "sta?", "stagnation"), + glob(false, "sta?n", "stagnation"), + + glob(true, "{abc,def}ghi", "defghi"), + glob(true, "{abc,abcd}a", "abcda"), + glob(true, "{a,ab}{bc,f}", "abc"), + glob(true, "{*,**}{a,b}", "ab"), + glob(false, "{*,**}{a,b}", "ac"), + + glob(true, "/{rate,[a-z][a-z][a-z]}*", "/rate"), + glob(true, "/{rate,[0-9][0-9][0-9]}*", "/rate"), + glob(true, "/{rate,[a-z][a-z][a-z]}*", "/usd"), + + glob(true, "{*.google.*,*.yandex.*}", "www.google.com", '.'), + glob(true, "{*.google.*,*.yandex.*}", "www.yandex.com", '.'), + glob(false, "{*.google.*,*.yandex.*}", "yandex.com", '.'), + glob(false, "{*.google.*,*.yandex.*}", "google.com", '.'), + + glob(true, "{*.google.*,yandex.*}", "www.google.com", '.'), + glob(true, "{*.google.*,yandex.*}", "yandex.com", '.'), + glob(false, "{*.google.*,yandex.*}", "www.yandex.com", '.'), + glob(false, "{*.google.*,yandex.*}", "google.com", '.'), + + glob(true, "*//{,*.}example.com", "https://www.example.com"), + glob(true, "*//{,*.}example.com", "http://example.com"), + glob(false, "*//{,*.}example.com", "http://example.com.net"), + + glob(true, pattern_all, fixture_all_match), + glob(false, pattern_all, fixture_all_mismatch), + + glob(true, pattern_plain, fixture_plain_match), + glob(false, pattern_plain, fixture_plain_mismatch), + + glob(true, pattern_multiple, fixture_multiple_match), + glob(false, pattern_multiple, fixture_multiple_mismatch), + + glob(true, pattern_alternatives, fixture_alternatives_match), + glob(false, pattern_alternatives, fixture_alternatives_mismatch), + + glob(true, pattern_alternatives_suffix, fixture_alternatives_suffix_first_match), + glob(false, pattern_alternatives_suffix, fixture_alternatives_suffix_first_mismatch), + glob(true, pattern_alternatives_suffix, fixture_alternatives_suffix_second), + + glob(true, pattern_alternatives_combine_hard, fixture_alternatives_combine_hard), + + glob(true, pattern_alternatives_combine_lite, fixture_alternatives_combine_lite), + + glob(true, pattern_prefix, fixture_prefix_suffix_match), + glob(false, pattern_prefix, fixture_prefix_suffix_mismatch), + + glob(true, pattern_suffix, fixture_prefix_suffix_match), + glob(false, pattern_suffix, fixture_prefix_suffix_mismatch), + + glob(true, pattern_prefix_suffix, fixture_prefix_suffix_match), + glob(false, pattern_prefix_suffix, fixture_prefix_suffix_mismatch), + } { + t.Run("", func(t *testing.T) { + g := MustCompile(test.pattern, test.delimiters...) + result := g.Match(test.match) + if result != test.should { + t.Errorf( + "pattern %q matching %q should be %v but got %v\n%s", + test.pattern, test.match, test.should, result, g, + ) + } + }) + } +} + +func TestQuoteMeta(t *testing.T) { + for id, test := range []struct { + in, out string + }{ + { + in: `[foo*]`, + out: `\[foo\*\]`, + }, + { + in: `{foo*}`, + out: `\{foo\*\}`, + }, + { + in: `*?\[]{}`, + out: `\*\?\\\[\]\{\}`, + }, + { + in: `some text and *?\[]{}`, + out: `some text and \*\?\\\[\]\{\}`, + }, + } { + act := QuoteMeta(test.in) + if act != test.out { + t.Errorf("#%d QuoteMeta(%q) = %q; want %q", id, test.in, act, test.out) + } + if _, err := Compile(act); err != nil { + t.Errorf("#%d _, err := Compile(QuoteMeta(%q) = %q); err = %q", id, test.in, act, err) + } + } +} + +func BenchmarkParseGlob(b *testing.B) { + for i := 0; i < b.N; i++ { + Compile(pattern_all) + } +} +func BenchmarkParseRegexp(b *testing.B) { + for i := 0; i < b.N; i++ { + regexp.MustCompile(regexp_all) + } +} + +func BenchmarkAllGlobMatch(b *testing.B) { + m, _ := Compile(pattern_all) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_all_match) + } +} +func BenchmarkAllGlobMatchParallel(b *testing.B) { + m, _ := Compile(pattern_all) + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _ = m.Match(fixture_all_match) + } + }) +} + +func BenchmarkAllRegexpMatch(b *testing.B) { + m := regexp.MustCompile(regexp_all) + f := []byte(fixture_all_match) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} +func BenchmarkAllGlobMismatch(b *testing.B) { + m, _ := Compile(pattern_all) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_all_mismatch) + } +} +func BenchmarkAllGlobMismatchParallel(b *testing.B) { + m, _ := Compile(pattern_all) + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _ = m.Match(fixture_all_mismatch) + } + }) +} +func BenchmarkAllRegexpMismatch(b *testing.B) { + m := regexp.MustCompile(regexp_all) + f := []byte(fixture_all_mismatch) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} + +func BenchmarkMultipleGlobMatch(b *testing.B) { + m, _ := Compile(pattern_multiple) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_multiple_match) + } +} +func BenchmarkMultipleRegexpMatch(b *testing.B) { + m := regexp.MustCompile(regexp_multiple) + f := []byte(fixture_multiple_match) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} +func BenchmarkMultipleGlobMismatch(b *testing.B) { + m, _ := Compile(pattern_multiple) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_multiple_mismatch) + } +} +func BenchmarkMultipleRegexpMismatch(b *testing.B) { + m := regexp.MustCompile(regexp_multiple) + f := []byte(fixture_multiple_mismatch) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} + +func BenchmarkAlternativesGlobMatch(b *testing.B) { + m, _ := Compile(pattern_alternatives) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_alternatives_match) + } +} +func BenchmarkAlternativesGlobMismatch(b *testing.B) { + m, _ := Compile(pattern_alternatives) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_alternatives_mismatch) + } +} +func BenchmarkAlternativesRegexpMatch(b *testing.B) { + m := regexp.MustCompile(regexp_alternatives) + f := []byte(fixture_alternatives_match) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} +func BenchmarkAlternativesRegexpMismatch(b *testing.B) { + m := regexp.MustCompile(regexp_alternatives) + f := []byte(fixture_alternatives_mismatch) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} + +func BenchmarkAlternativesSuffixFirstGlobMatch(b *testing.B) { + m, _ := Compile(pattern_alternatives_suffix) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_alternatives_suffix_first_match) + } +} +func BenchmarkAlternativesSuffixFirstGlobMismatch(b *testing.B) { + m, _ := Compile(pattern_alternatives_suffix) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_alternatives_suffix_first_mismatch) + } +} +func BenchmarkAlternativesSuffixSecondGlobMatch(b *testing.B) { + m, _ := Compile(pattern_alternatives_suffix) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_alternatives_suffix_second) + } +} +func BenchmarkAlternativesCombineLiteGlobMatch(b *testing.B) { + m, _ := Compile(pattern_alternatives_combine_lite) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_alternatives_combine_lite) + } +} +func BenchmarkAlternativesCombineHardGlobMatch(b *testing.B) { + m, _ := Compile(pattern_alternatives_combine_hard) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_alternatives_combine_hard) + } +} +func BenchmarkAlternativesSuffixFirstRegexpMatch(b *testing.B) { + m := regexp.MustCompile(regexp_alternatives_suffix) + f := []byte(fixture_alternatives_suffix_first_match) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} +func BenchmarkAlternativesSuffixFirstRegexpMismatch(b *testing.B) { + m := regexp.MustCompile(regexp_alternatives_suffix) + f := []byte(fixture_alternatives_suffix_first_mismatch) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} +func BenchmarkAlternativesSuffixSecondRegexpMatch(b *testing.B) { + m := regexp.MustCompile(regexp_alternatives_suffix) + f := []byte(fixture_alternatives_suffix_second) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} +func BenchmarkAlternativesCombineLiteRegexpMatch(b *testing.B) { + m := regexp.MustCompile(regexp_alternatives_combine_lite) + f := []byte(fixture_alternatives_combine_lite) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} +func BenchmarkAlternativesCombineHardRegexpMatch(b *testing.B) { + m := regexp.MustCompile(regexp_alternatives_combine_hard) + f := []byte(fixture_alternatives_combine_hard) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} + +func BenchmarkPlainGlobMatch(b *testing.B) { + m, _ := Compile(pattern_plain) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_plain_match) + } +} +func BenchmarkPlainRegexpMatch(b *testing.B) { + m := regexp.MustCompile(regexp_plain) + f := []byte(fixture_plain_match) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} +func BenchmarkPlainGlobMismatch(b *testing.B) { + m, _ := Compile(pattern_plain) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_plain_mismatch) + } +} +func BenchmarkPlainRegexpMismatch(b *testing.B) { + m := regexp.MustCompile(regexp_plain) + f := []byte(fixture_plain_mismatch) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} + +func BenchmarkPrefixGlobMatch(b *testing.B) { + m, _ := Compile(pattern_prefix) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_prefix_suffix_match) + } +} +func BenchmarkPrefixRegexpMatch(b *testing.B) { + m := regexp.MustCompile(regexp_prefix) + f := []byte(fixture_prefix_suffix_match) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} +func BenchmarkPrefixGlobMismatch(b *testing.B) { + m, _ := Compile(pattern_prefix) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_prefix_suffix_mismatch) + } +} +func BenchmarkPrefixRegexpMismatch(b *testing.B) { + m := regexp.MustCompile(regexp_prefix) + f := []byte(fixture_prefix_suffix_mismatch) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} + +func BenchmarkSuffixGlobMatch(b *testing.B) { + m, _ := Compile(pattern_suffix) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_prefix_suffix_match) + } +} +func BenchmarkSuffixRegexpMatch(b *testing.B) { + m := regexp.MustCompile(regexp_suffix) + f := []byte(fixture_prefix_suffix_match) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} +func BenchmarkSuffixGlobMismatch(b *testing.B) { + m, _ := Compile(pattern_suffix) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_prefix_suffix_mismatch) + } +} +func BenchmarkSuffixRegexpMismatch(b *testing.B) { + m := regexp.MustCompile(regexp_suffix) + f := []byte(fixture_prefix_suffix_mismatch) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} + +func BenchmarkPrefixSuffixGlobMatch(b *testing.B) { + m, _ := Compile(pattern_prefix_suffix) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_prefix_suffix_match) + } +} +func BenchmarkPrefixSuffixRegexpMatch(b *testing.B) { + m := regexp.MustCompile(regexp_prefix_suffix) + f := []byte(fixture_prefix_suffix_match) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} +func BenchmarkPrefixSuffixGlobMismatch(b *testing.B) { + m, _ := Compile(pattern_prefix_suffix) + + for i := 0; i < b.N; i++ { + _ = m.Match(fixture_prefix_suffix_mismatch) + } +} +func BenchmarkPrefixSuffixRegexpMismatch(b *testing.B) { + m := regexp.MustCompile(regexp_prefix_suffix) + f := []byte(fixture_prefix_suffix_mismatch) + + for i := 0; i < b.N; i++ { + _ = m.Match(f) + } +} diff --git a/vendor/github.com/gobwas/glob/match/any.go b/vendor/github.com/gobwas/glob/match/any.go new file mode 100644 index 0000000..514a9a5 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/any.go @@ -0,0 +1,45 @@ +package match + +import ( + "fmt" + "github.com/gobwas/glob/util/strings" +) + +type Any struct { + Separators []rune +} + +func NewAny(s []rune) Any { + return Any{s} +} + +func (self Any) Match(s string) bool { + return strings.IndexAnyRunes(s, self.Separators) == -1 +} + +func (self Any) Index(s string) (int, []int) { + found := strings.IndexAnyRunes(s, self.Separators) + switch found { + case -1: + case 0: + return 0, segments0 + default: + s = s[:found] + } + + segments := acquireSegments(len(s)) + for i := range s { + segments = append(segments, i) + } + segments = append(segments, len(s)) + + return 0, segments +} + +func (self Any) Len() int { + return lenNo +} + +func (self Any) String() string { + return fmt.Sprintf("
+
+
+### Backers
+
+Thank you to all our backers! [[Become a backer](https://opencollective.com/colly#backer)]
+
+
+
+
+### Sponsors
+
+Thank you to all our sponsors! (please ask your company to also support this open source project by [becoming a sponsor](https://opencollective.com/colly#sponsor))
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/vendor/github.com/gocolly/colly/LICENSE.txt b/vendor/github.com/gocolly/colly/LICENSE.txt
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/vendor/github.com/gocolly/colly/LICENSE.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/vendor/github.com/gocolly/colly/README.md b/vendor/github.com/gocolly/colly/README.md
new file mode 100644
index 0000000..783031f
--- /dev/null
+++ b/vendor/github.com/gocolly/colly/README.md
@@ -0,0 +1,111 @@
+# Colly
+
+Lightning Fast and Elegant Scraping Framework for Gophers
+
+Colly provides a clean interface to write any kind of crawler/scraper/spider.
+
+With Colly you can easily extract structured data from websites, which can be used for a wide range of applications, like data mining, data processing or archiving.
+
+[](https://godoc.org/github.com/gocolly/colly)
+[](#backers) [](#sponsors) [](https://travis-ci.org/gocolly/colly)
+[](http://goreportcard.com/report/gocolly/colly)
+[](https://github.com/gocolly/colly/tree/master/_examples)
+[](https://codecov.io/github/gocolly/colly?branch=master)
+[](https://app.fossa.io/projects/git%2Bgithub.com%2Fgocolly%2Fcolly?ref=badge_shield)
+[](https://twitter.com/gocolly)
+
+
+## Features
+
+ * Clean API
+ * Fast (>1k request/sec on a single core)
+ * Manages request delays and maximum concurrency per domain
+ * Automatic cookie and session handling
+ * Sync/async/parallel scraping
+ * Caching
+ * Automatic encoding of non-unicode responses
+ * Robots.txt support
+ * Distributed scraping
+ * Configuration via environment variables
+ * Extensions
+
+
+## Example
+
+```go
+func main() {
+ c := colly.NewCollector()
+
+ // Find and visit all links
+ c.OnHTML("a[href]", func(e *colly.HTMLElement) {
+ e.Request.Visit(e.Attr("href"))
+ })
+
+ c.OnRequest(func(r *colly.Request) {
+ fmt.Println("Visiting", r.URL)
+ })
+
+ c.Visit("http://go-colly.org/")
+}
+```
+
+See [examples folder](https://github.com/gocolly/colly/tree/master/_examples) for more detailed examples.
+
+
+## Installation
+
+```
+go get -u github.com/gocolly/colly/...
+```
+
+
+## Bugs
+
+Bugs or suggestions? Visit the [issue tracker](https://github.com/gocolly/colly/issues) or join `#colly` on freenode
+
+
+## Other Projects Using Colly
+
+Below is a list of public, open source projects that use Colly:
+
+ * [greenpeace/check-my-pages](https://github.com/greenpeace/check-my-pages) Scraping script to test the Spanish Greenpeace web archive
+ * [altsab/gowap](https://github.com/altsab/gowap) Wappalyzer implementation in Go
+ * [jesuiscamille/goquotes](https://github.com/jesuiscamille/goquotes) A quotes scrapper, making your day a little better!
+ * [jivesearch/jivesearch](https://github.com/jivesearch/jivesearch) A search engine that doesn't track you.
+
+
+If you are using Colly in a project please send a pull request to add it to the list.
+
+## Contributors
+
+This project exists thanks to all the people who contribute. [[Contribute]](CONTRIBUTING.md).
+
+
+
+## Backers
+
+Thank you to all our backers! 🙠[[Become a backer](https://opencollective.com/colly#backer)]
+
+
+
+
+## Sponsors
+
+Support this project by becoming a sponsor. Your logo will show up here with a link to your website. [[Become a sponsor](https://opencollective.com/colly#sponsor)]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+## License
+[](https://app.fossa.io/projects/git%2Bgithub.com%2Fgocolly%2Fcolly?ref=badge_large)
diff --git a/vendor/github.com/gocolly/colly/VERSION b/vendor/github.com/gocolly/colly/VERSION
new file mode 100644
index 0000000..9084fa2
--- /dev/null
+++ b/vendor/github.com/gocolly/colly/VERSION
@@ -0,0 +1 @@
+1.1.0
diff --git a/vendor/github.com/gocolly/colly/colly.go b/vendor/github.com/gocolly/colly/colly.go
new file mode 100644
index 0000000..131660a
--- /dev/null
+++ b/vendor/github.com/gocolly/colly/colly.go
@@ -0,0 +1,1281 @@
+// Copyright 2018 Adam Tauber
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package colly implements a HTTP scraping framework
+package colly
+
+import (
+ "bytes"
+ "context"
+ "crypto/rand"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "hash/fnv"
+ "io"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "net/http/cookiejar"
+ "net/url"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "golang.org/x/net/html"
+ "google.golang.org/appengine/urlfetch"
+
+ "github.com/PuerkitoBio/goquery"
+ "github.com/antchfx/htmlquery"
+ "github.com/antchfx/xmlquery"
+ "github.com/kennygrant/sanitize"
+ "github.com/temoto/robotstxt"
+
+ "github.com/gocolly/colly/debug"
+ "github.com/gocolly/colly/storage"
+)
+
+// Collector provides the scraper instance for a scraping job
+type Collector struct {
+ // UserAgent is the User-Agent string used by HTTP requests
+ UserAgent string
+ // MaxDepth limits the recursion depth of visited URLs.
+ // Set it to 0 for infinite recursion (default).
+ MaxDepth int
+ // AllowedDomains is a domain whitelist.
+ // Leave it blank to allow any domains to be visited
+ AllowedDomains []string
+ // DisallowedDomains is a domain blacklist.
+ DisallowedDomains []string
+ // DisallowedURLFilters is a list of regular expressions which restricts
+ // visiting URLs. If any of the rules matches to a URL the
+ // request will be stopped. DisallowedURLFilters will
+ // be evaluated before URLFilters
+ // Leave it blank to allow any URLs to be visited
+ DisallowedURLFilters []*regexp.Regexp
+ // URLFilters is a list of regular expressions which restricts
+ // visiting URLs. If any of the rules matches to a URL the
+ // request won't be stopped. DisallowedURLFilters will
+ // be evaluated before URLFilters
+
+ // Leave it blank to allow any URLs to be visited
+ URLFilters []*regexp.Regexp
+
+ // AllowURLRevisit allows multiple downloads of the same URL
+ AllowURLRevisit bool
+ // MaxBodySize is the limit of the retrieved response body in bytes.
+ // 0 means unlimited.
+ // The default value for MaxBodySize is 10MB (10 * 1024 * 1024 bytes).
+ MaxBodySize int
+ // CacheDir specifies a location where GET requests are cached as files.
+ // When it's not defined, caching is disabled.
+ CacheDir string
+ // IgnoreRobotsTxt allows the Collector to ignore any restrictions set by
+ // the target host's robots.txt file. See http://www.robotstxt.org/ for more
+ // information.
+ IgnoreRobotsTxt bool
+ // Async turns on asynchronous network communication. Use Collector.Wait() to
+ // be sure all requests have been finished.
+ Async bool
+ // ParseHTTPErrorResponse allows parsing HTTP responses with non 2xx status codes.
+ // By default, Colly parses only successful HTTP responses. Set ParseHTTPErrorResponse
+ // to true to enable it.
+ ParseHTTPErrorResponse bool
+ // ID is the unique identifier of a collector
+ ID uint32
+ // DetectCharset can enable character encoding detection for non-utf8 response bodies
+ // without explicit charset declaration. This feature uses https://github.com/saintfish/chardet
+ DetectCharset bool
+ // RedirectHandler allows control on how a redirect will be managed
+ RedirectHandler func(req *http.Request, via []*http.Request) error
+ store storage.Storage
+ debugger debug.Debugger
+ robotsMap map[string]*robotstxt.RobotsData
+ htmlCallbacks []*htmlCallbackContainer
+ xmlCallbacks []*xmlCallbackContainer
+ requestCallbacks []RequestCallback
+ responseCallbacks []ResponseCallback
+ errorCallbacks []ErrorCallback
+ scrapedCallbacks []ScrapedCallback
+ requestCount uint32
+ responseCount uint32
+ backend *httpBackend
+ wg *sync.WaitGroup
+ lock *sync.RWMutex
+}
+
+// RequestCallback is a type alias for OnRequest callback functions
+type RequestCallback func(*Request)
+
+// ResponseCallback is a type alias for OnResponse callback functions
+type ResponseCallback func(*Response)
+
+// HTMLCallback is a type alias for OnHTML callback functions
+type HTMLCallback func(*HTMLElement)
+
+// XMLCallback is a type alias for OnXML callback functions
+type XMLCallback func(*XMLElement)
+
+// ErrorCallback is a type alias for OnError callback functions
+type ErrorCallback func(*Response, error)
+
+// ScrapedCallback is a type alias for OnScraped callback functions
+type ScrapedCallback func(*Response)
+
+// ProxyFunc is a type alias for proxy setter functions.
+type ProxyFunc func(*http.Request) (*url.URL, error)
+
+type htmlCallbackContainer struct {
+ Selector string
+ Function HTMLCallback
+}
+
+type xmlCallbackContainer struct {
+ Query string
+ Function XMLCallback
+}
+
+type cookieJarSerializer struct {
+ store storage.Storage
+ lock *sync.RWMutex
+}
+
+var collectorCounter uint32
+
+// The key type is unexported to prevent collisions with context keys defined in
+// other packages.
+type key int
+
+// ProxyURLKey is the context key for the request proxy address.
+const ProxyURLKey key = iota
+
+var (
+ // ErrForbiddenDomain is the error thrown if visiting
+ // a domain which is not allowed in AllowedDomains
+ ErrForbiddenDomain = errors.New("Forbidden domain")
+ // ErrMissingURL is the error type for missing URL errors
+ ErrMissingURL = errors.New("Missing URL")
+ // ErrMaxDepth is the error type for exceeding max depth
+ ErrMaxDepth = errors.New("Max depth limit reached")
+ // ErrForbiddenURL is the error thrown if visiting
+ // a URL which is not allowed by URLFilters
+ ErrForbiddenURL = errors.New("ForbiddenURL")
+
+ // ErrNoURLFiltersMatch is the error thrown if visiting
+ // a URL which is not allowed by URLFilters
+ ErrNoURLFiltersMatch = errors.New("No URLFilters match")
+ // ErrAlreadyVisited is the error type for already visited URLs
+ ErrAlreadyVisited = errors.New("URL already visited")
+ // ErrRobotsTxtBlocked is the error type for robots.txt errors
+ ErrRobotsTxtBlocked = errors.New("URL blocked by robots.txt")
+ // ErrNoCookieJar is the error type for missing cookie jar
+ ErrNoCookieJar = errors.New("Cookie jar is not available")
+ // ErrNoPattern is the error type for LimitRules without patterns
+ ErrNoPattern = errors.New("No pattern defined in LimitRule")
+)
+
+var envMap = map[string]func(*Collector, string){
+ "ALLOWED_DOMAINS": func(c *Collector, val string) {
+ c.AllowedDomains = strings.Split(val, ",")
+ },
+ "CACHE_DIR": func(c *Collector, val string) {
+ c.CacheDir = val
+ },
+ "DETECT_CHARSET": func(c *Collector, val string) {
+ c.DetectCharset = isYesString(val)
+ },
+ "DISABLE_COOKIES": func(c *Collector, _ string) {
+ c.backend.Client.Jar = nil
+ },
+ "DISALLOWED_DOMAINS": func(c *Collector, val string) {
+ c.DisallowedDomains = strings.Split(val, ",")
+ },
+ "IGNORE_ROBOTSTXT": func(c *Collector, val string) {
+ c.IgnoreRobotsTxt = isYesString(val)
+ },
+ "FOLLOW_REDIRECTS": func(c *Collector, val string) {
+ if !isYesString(val) {
+ c.RedirectHandler = func(req *http.Request, via []*http.Request) error {
+ return http.ErrUseLastResponse
+ }
+ }
+ },
+ "MAX_BODY_SIZE": func(c *Collector, val string) {
+ size, err := strconv.Atoi(val)
+ if err == nil {
+ c.MaxBodySize = size
+ }
+ },
+ "MAX_DEPTH": func(c *Collector, val string) {
+ maxDepth, err := strconv.Atoi(val)
+ if err != nil {
+ c.MaxDepth = maxDepth
+ }
+ },
+ "PARSE_HTTP_ERROR_RESPONSE": func(c *Collector, val string) {
+ c.ParseHTTPErrorResponse = isYesString(val)
+ },
+ "USER_AGENT": func(c *Collector, val string) {
+ c.UserAgent = val
+ },
+}
+
+// NewCollector creates a new Collector instance with default configuration
+func NewCollector(options ...func(*Collector)) *Collector {
+ c := &Collector{}
+ c.Init()
+
+ for _, f := range options {
+ f(c)
+ }
+
+ c.parseSettingsFromEnv()
+
+ return c
+}
+
+// UserAgent sets the user agent used by the Collector.
+func UserAgent(ua string) func(*Collector) {
+ return func(c *Collector) {
+ c.UserAgent = ua
+ }
+}
+
+// MaxDepth limits the recursion depth of visited URLs.
+func MaxDepth(depth int) func(*Collector) {
+ return func(c *Collector) {
+ c.MaxDepth = depth
+ }
+}
+
+// AllowedDomains sets the domain whitelist used by the Collector.
+func AllowedDomains(domains ...string) func(*Collector) {
+ return func(c *Collector) {
+ c.AllowedDomains = domains
+ }
+}
+
+// ParseHTTPErrorResponse allows parsing responses with HTTP errors
+func ParseHTTPErrorResponse() func(*Collector) {
+ return func(c *Collector) {
+ c.ParseHTTPErrorResponse = true
+ }
+}
+
+// DisallowedDomains sets the domain blacklist used by the Collector.
+func DisallowedDomains(domains ...string) func(*Collector) {
+ return func(c *Collector) {
+ c.DisallowedDomains = domains
+ }
+}
+
+// DisallowedURLFilters sets the list of regular expressions which restricts
+// visiting URLs. If any of the rules matches to a URL the request will be stopped.
+func DisallowedURLFilters(filters ...*regexp.Regexp) func(*Collector) {
+ return func(c *Collector) {
+ c.DisallowedURLFilters = filters
+ }
+}
+
+// URLFilters sets the list of regular expressions which restricts
+// visiting URLs. If any of the rules matches to a URL the request won't be stopped.
+func URLFilters(filters ...*regexp.Regexp) func(*Collector) {
+ return func(c *Collector) {
+ c.URLFilters = filters
+ }
+}
+
+// AllowURLRevisit instructs the Collector to allow multiple downloads of the same URL
+func AllowURLRevisit() func(*Collector) {
+ return func(c *Collector) {
+ c.AllowURLRevisit = true
+ }
+}
+
+// MaxBodySize sets the limit of the retrieved response body in bytes.
+func MaxBodySize(sizeInBytes int) func(*Collector) {
+ return func(c *Collector) {
+ c.MaxBodySize = sizeInBytes
+ }
+}
+
+// CacheDir specifies the location where GET requests are cached as files.
+func CacheDir(path string) func(*Collector) {
+ return func(c *Collector) {
+ c.CacheDir = path
+ }
+}
+
+// IgnoreRobotsTxt instructs the Collector to ignore any restrictions
+// set by the target host's robots.txt file.
+func IgnoreRobotsTxt() func(*Collector) {
+ return func(c *Collector) {
+ c.IgnoreRobotsTxt = true
+ }
+}
+
+// ID sets the unique identifier of the Collector.
+func ID(id uint32) func(*Collector) {
+ return func(c *Collector) {
+ c.ID = id
+ }
+}
+
+// Async turns on asynchronous network requests.
+func Async(a bool) func(*Collector) {
+ return func(c *Collector) {
+ c.Async = a
+ }
+}
+
+// DetectCharset enables character encoding detection for non-utf8 response bodies
+// without explicit charset declaration. This feature uses https://github.com/saintfish/chardet
+func DetectCharset() func(*Collector) {
+ return func(c *Collector) {
+ c.DetectCharset = true
+ }
+}
+
+// Debugger sets the debugger used by the Collector.
+func Debugger(d debug.Debugger) func(*Collector) {
+ return func(c *Collector) {
+ d.Init()
+ c.debugger = d
+ }
+}
+
+// Init initializes the Collector's private variables and sets default
+// configuration for the Collector
+func (c *Collector) Init() {
+ c.UserAgent = "colly - https://github.com/gocolly/colly"
+ c.MaxDepth = 0
+ c.store = &storage.InMemoryStorage{}
+ c.store.Init()
+ c.MaxBodySize = 10 * 1024 * 1024
+ c.backend = &httpBackend{}
+ jar, _ := cookiejar.New(nil)
+ c.backend.Init(jar)
+ c.backend.Client.CheckRedirect = c.checkRedirectFunc()
+ c.wg = &sync.WaitGroup{}
+ c.lock = &sync.RWMutex{}
+ c.robotsMap = make(map[string]*robotstxt.RobotsData)
+ c.IgnoreRobotsTxt = true
+ c.ID = atomic.AddUint32(&collectorCounter, 1)
+}
+
+// Appengine will replace the Collector's backend http.Client
+// With an Http.Client that is provided by appengine/urlfetch
+// This function should be used when the scraper is run on
+// Google App Engine. Example:
+// func startScraper(w http.ResponseWriter, r *http.Request) {
+// ctx := appengine.NewContext(r)
+// c := colly.NewCollector()
+// c.Appengine(ctx)
+// ...
+// c.Visit("https://google.ca")
+// }
+func (c *Collector) Appengine(ctx context.Context) {
+ client := urlfetch.Client(ctx)
+ client.Jar = c.backend.Client.Jar
+ client.CheckRedirect = c.backend.Client.CheckRedirect
+ client.Timeout = c.backend.Client.Timeout
+
+ c.backend.Client = client
+}
+
+// Visit starts Collector's collecting job by creating a
+// request to the URL specified in parameter.
+// Visit also calls the previously provided callbacks
+func (c *Collector) Visit(URL string) error {
+ return c.scrape(URL, "GET", 1, nil, nil, nil, true)
+}
+
+// Post starts a collector job by creating a POST request.
+// Post also calls the previously provided callbacks
+func (c *Collector) Post(URL string, requestData map[string]string) error {
+ return c.scrape(URL, "POST", 1, createFormReader(requestData), nil, nil, true)
+}
+
+// PostRaw starts a collector job by creating a POST request with raw binary data.
+// Post also calls the previously provided callbacks
+func (c *Collector) PostRaw(URL string, requestData []byte) error {
+ return c.scrape(URL, "POST", 1, bytes.NewReader(requestData), nil, nil, true)
+}
+
+// PostMultipart starts a collector job by creating a Multipart POST request
+// with raw binary data. PostMultipart also calls the previously provided callbacks
+func (c *Collector) PostMultipart(URL string, requestData map[string][]byte) error {
+ boundary := randomBoundary()
+ hdr := http.Header{}
+ hdr.Set("Content-Type", "multipart/form-data; boundary="+boundary)
+ hdr.Set("User-Agent", c.UserAgent)
+ return c.scrape(URL, "POST", 1, createMultipartReader(boundary, requestData), nil, hdr, true)
+}
+
+// Request starts a collector job by creating a custom HTTP request
+// where method, context, headers and request data can be specified.
+// Set requestData, ctx, hdr parameters to nil if you don't want to use them.
+// Valid methods:
+// - "GET"
+// - "POST"
+// - "PUT"
+// - "DELETE"
+// - "PATCH"
+// - "OPTIONS"
+func (c *Collector) Request(method, URL string, requestData io.Reader, ctx *Context, hdr http.Header) error {
+ return c.scrape(URL, method, 1, requestData, ctx, hdr, true)
+}
+
+// SetDebugger attaches a debugger to the collector
+func (c *Collector) SetDebugger(d debug.Debugger) {
+ d.Init()
+ c.debugger = d
+}
+
+// UnmarshalRequest creates a Request from serialized data
+func (c *Collector) UnmarshalRequest(r []byte) (*Request, error) {
+ req := &serializableRequest{}
+ err := json.Unmarshal(r, req)
+ if err != nil {
+ return nil, err
+ }
+
+ u, err := url.Parse(req.URL)
+ if err != nil {
+ return nil, err
+ }
+
+ ctx := NewContext()
+ for k, v := range req.Ctx {
+ ctx.Put(k, v)
+ }
+
+ return &Request{
+ Method: req.Method,
+ URL: u,
+ Body: bytes.NewReader(req.Body),
+ Ctx: ctx,
+ ID: atomic.AddUint32(&c.requestCount, 1),
+ Headers: &req.Headers,
+ collector: c,
+ }, nil
+}
+
+func (c *Collector) scrape(u, method string, depth int, requestData io.Reader, ctx *Context, hdr http.Header, checkRevisit bool) error {
+ if err := c.requestCheck(u, method, depth, checkRevisit); err != nil {
+ return err
+ }
+ parsedURL, err := url.Parse(u)
+ if err != nil {
+ return err
+ }
+ if parsedURL.Scheme == "" {
+ parsedURL.Scheme = "http"
+ }
+ if !c.isDomainAllowed(parsedURL.Host) {
+ return ErrForbiddenDomain
+ }
+ if !c.IgnoreRobotsTxt {
+ if err = c.checkRobots(parsedURL); err != nil {
+ return err
+ }
+ }
+ if hdr == nil {
+ hdr = http.Header{"User-Agent": []string{c.UserAgent}}
+ }
+ rc, ok := requestData.(io.ReadCloser)
+ if !ok && requestData != nil {
+ rc = ioutil.NopCloser(requestData)
+ }
+ req := &http.Request{
+ Method: method,
+ URL: parsedURL,
+ Proto: "HTTP/1.1",
+ ProtoMajor: 1,
+ ProtoMinor: 1,
+ Header: hdr,
+ Body: rc,
+ Host: parsedURL.Host,
+ }
+ setRequestBody(req, requestData)
+ u = parsedURL.String()
+ c.wg.Add(1)
+ if c.Async {
+ go c.fetch(u, method, depth, requestData, ctx, hdr, req)
+ return nil
+ }
+ return c.fetch(u, method, depth, requestData, ctx, hdr, req)
+}
+
+func setRequestBody(req *http.Request, body io.Reader) {
+ if body != nil {
+ switch v := body.(type) {
+ case *bytes.Buffer:
+ req.ContentLength = int64(v.Len())
+ buf := v.Bytes()
+ req.GetBody = func() (io.ReadCloser, error) {
+ r := bytes.NewReader(buf)
+ return ioutil.NopCloser(r), nil
+ }
+ case *bytes.Reader:
+ req.ContentLength = int64(v.Len())
+ snapshot := *v
+ req.GetBody = func() (io.ReadCloser, error) {
+ r := snapshot
+ return ioutil.NopCloser(&r), nil
+ }
+ case *strings.Reader:
+ req.ContentLength = int64(v.Len())
+ snapshot := *v
+ req.GetBody = func() (io.ReadCloser, error) {
+ r := snapshot
+ return ioutil.NopCloser(&r), nil
+ }
+ }
+ if req.GetBody != nil && req.ContentLength == 0 {
+ req.Body = http.NoBody
+ req.GetBody = func() (io.ReadCloser, error) { return http.NoBody, nil }
+ }
+ }
+}
+
+func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ctx *Context, hdr http.Header, req *http.Request) error {
+ defer c.wg.Done()
+ if ctx == nil {
+ ctx = NewContext()
+ }
+ request := &Request{
+ URL: req.URL,
+ Headers: &req.Header,
+ Ctx: ctx,
+ Depth: depth,
+ Method: method,
+ Body: requestData,
+ collector: c,
+ ID: atomic.AddUint32(&c.requestCount, 1),
+ }
+
+ c.handleOnRequest(request)
+
+ if request.abort {
+ return nil
+ }
+
+ if method == "POST" && req.Header.Get("Content-Type") == "" {
+ req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+ }
+
+ if req.Header.Get("Accept") == "" {
+ req.Header.Set("Accept", "*/*")
+ }
+
+ origURL := req.URL
+ response, err := c.backend.Cache(req, c.MaxBodySize, c.CacheDir)
+ if err := c.handleOnError(response, err, request, ctx); err != nil {
+ return err
+ }
+ if req.URL != origURL {
+ request.URL = req.URL
+ request.Headers = &req.Header
+ }
+ if proxyURL, ok := req.Context().Value(ProxyURLKey).(string); ok {
+ request.ProxyURL = proxyURL
+ }
+ atomic.AddUint32(&c.responseCount, 1)
+ response.Ctx = ctx
+ response.Request = request
+
+ err = response.fixCharset(c.DetectCharset, request.ResponseCharacterEncoding)
+ if err != nil {
+ return err
+ }
+
+ c.handleOnResponse(response)
+
+ err = c.handleOnHTML(response)
+ if err != nil {
+ c.handleOnError(response, err, request, ctx)
+ }
+
+ err = c.handleOnXML(response)
+ if err != nil {
+ c.handleOnError(response, err, request, ctx)
+ }
+
+ c.handleOnScraped(response)
+
+ return err
+}
+
+func (c *Collector) requestCheck(u, method string, depth int, checkRevisit bool) error {
+ if u == "" {
+ return ErrMissingURL
+ }
+ if c.MaxDepth > 0 && c.MaxDepth < depth {
+ return ErrMaxDepth
+ }
+ if len(c.DisallowedURLFilters) > 0 {
+ if isMatchingFilter(c.DisallowedURLFilters, []byte(u)) {
+ return ErrForbiddenURL
+ }
+ }
+ if len(c.URLFilters) > 0 {
+ if !isMatchingFilter(c.URLFilters, []byte(u)) {
+ return ErrNoURLFiltersMatch
+ }
+ }
+ if checkRevisit && !c.AllowURLRevisit && method == "GET" {
+ h := fnv.New64a()
+ h.Write([]byte(u))
+ uHash := h.Sum64()
+ visited, err := c.store.IsVisited(uHash)
+ if err != nil {
+ return err
+ }
+ if visited {
+ return ErrAlreadyVisited
+ }
+ return c.store.Visited(uHash)
+ }
+ return nil
+}
+
+func (c *Collector) isDomainAllowed(domain string) bool {
+ for _, d2 := range c.DisallowedDomains {
+ if d2 == domain {
+ return false
+ }
+ }
+ if c.AllowedDomains == nil || len(c.AllowedDomains) == 0 {
+ return true
+ }
+ for _, d2 := range c.AllowedDomains {
+ if d2 == domain {
+ return true
+ }
+ }
+ return false
+}
+
+func (c *Collector) checkRobots(u *url.URL) error {
+ c.lock.RLock()
+ robot, ok := c.robotsMap[u.Host]
+ c.lock.RUnlock()
+
+ if !ok {
+ // no robots file cached
+ resp, err := c.backend.Client.Get(u.Scheme + "://" + u.Host + "/robots.txt")
+ if err != nil {
+ return err
+ }
+ robot, err = robotstxt.FromResponse(resp)
+ if err != nil {
+ return err
+ }
+ c.lock.Lock()
+ c.robotsMap[u.Host] = robot
+ c.lock.Unlock()
+ }
+
+ uaGroup := robot.FindGroup(c.UserAgent)
+ if uaGroup == nil {
+ return nil
+ }
+
+ if !uaGroup.Test(u.EscapedPath()) {
+ return ErrRobotsTxtBlocked
+ }
+ return nil
+}
+
+// String is the text representation of the collector.
+// It contains useful debug information about the collector's internals
+func (c *Collector) String() string {
+ return fmt.Sprintf(
+ "Requests made: %d (%d responses) | Callbacks: OnRequest: %d, OnHTML: %d, OnResponse: %d, OnError: %d",
+ c.requestCount,
+ c.responseCount,
+ len(c.requestCallbacks),
+ len(c.htmlCallbacks),
+ len(c.responseCallbacks),
+ len(c.errorCallbacks),
+ )
+}
+
+// Wait returns when the collector jobs are finished
+func (c *Collector) Wait() {
+ c.wg.Wait()
+}
+
+// OnRequest registers a function. Function will be executed on every
+// request made by the Collector
+func (c *Collector) OnRequest(f RequestCallback) {
+ c.lock.Lock()
+ if c.requestCallbacks == nil {
+ c.requestCallbacks = make([]RequestCallback, 0, 4)
+ }
+ c.requestCallbacks = append(c.requestCallbacks, f)
+ c.lock.Unlock()
+}
+
+// OnResponse registers a function. Function will be executed on every response
+func (c *Collector) OnResponse(f ResponseCallback) {
+ c.lock.Lock()
+ if c.responseCallbacks == nil {
+ c.responseCallbacks = make([]ResponseCallback, 0, 4)
+ }
+ c.responseCallbacks = append(c.responseCallbacks, f)
+ c.lock.Unlock()
+}
+
+// OnHTML registers a function. Function will be executed on every HTML
+// element matched by the GoQuery Selector parameter.
+// GoQuery Selector is a selector used by https://github.com/PuerkitoBio/goquery
+func (c *Collector) OnHTML(goquerySelector string, f HTMLCallback) {
+ c.lock.Lock()
+ if c.htmlCallbacks == nil {
+ c.htmlCallbacks = make([]*htmlCallbackContainer, 0, 4)
+ }
+ c.htmlCallbacks = append(c.htmlCallbacks, &htmlCallbackContainer{
+ Selector: goquerySelector,
+ Function: f,
+ })
+ c.lock.Unlock()
+}
+
+// OnXML registers a function. Function will be executed on every XML
+// element matched by the xpath Query parameter.
+// xpath Query is used by https://github.com/antchfx/xmlquery
+func (c *Collector) OnXML(xpathQuery string, f XMLCallback) {
+ c.lock.Lock()
+ if c.xmlCallbacks == nil {
+ c.xmlCallbacks = make([]*xmlCallbackContainer, 0, 4)
+ }
+ c.xmlCallbacks = append(c.xmlCallbacks, &xmlCallbackContainer{
+ Query: xpathQuery,
+ Function: f,
+ })
+ c.lock.Unlock()
+}
+
+// OnHTMLDetach deregister a function. Function will not be execute after detached
+func (c *Collector) OnHTMLDetach(goquerySelector string) {
+ c.lock.Lock()
+ deleteIdx := -1
+ for i, cc := range c.htmlCallbacks {
+ if cc.Selector == goquerySelector {
+ deleteIdx = i
+ break
+ }
+ }
+ if deleteIdx != -1 {
+ c.htmlCallbacks = append(c.htmlCallbacks[:deleteIdx], c.htmlCallbacks[deleteIdx+1:]...)
+ }
+ c.lock.Unlock()
+}
+
+// OnXMLDetach deregister a function. Function will not be execute after detached
+func (c *Collector) OnXMLDetach(xpathQuery string) {
+ c.lock.Lock()
+ deleteIdx := -1
+ for i, cc := range c.xmlCallbacks {
+ if cc.Query == xpathQuery {
+ deleteIdx = i
+ break
+ }
+ }
+ if deleteIdx != -1 {
+ c.xmlCallbacks = append(c.xmlCallbacks[:deleteIdx], c.xmlCallbacks[deleteIdx+1:]...)
+ }
+ c.lock.Unlock()
+}
+
+// OnError registers a function. Function will be executed if an error
+// occurs during the HTTP request.
+func (c *Collector) OnError(f ErrorCallback) {
+ c.lock.Lock()
+ if c.errorCallbacks == nil {
+ c.errorCallbacks = make([]ErrorCallback, 0, 4)
+ }
+ c.errorCallbacks = append(c.errorCallbacks, f)
+ c.lock.Unlock()
+}
+
+// OnScraped registers a function. Function will be executed after
+// OnHTML, as a final part of the scraping.
+func (c *Collector) OnScraped(f ScrapedCallback) {
+ c.lock.Lock()
+ if c.scrapedCallbacks == nil {
+ c.scrapedCallbacks = make([]ScrapedCallback, 0, 4)
+ }
+ c.scrapedCallbacks = append(c.scrapedCallbacks, f)
+ c.lock.Unlock()
+}
+
+// WithTransport allows you to set a custom http.RoundTripper (transport)
+func (c *Collector) WithTransport(transport http.RoundTripper) {
+ c.backend.Client.Transport = transport
+}
+
+// DisableCookies turns off cookie handling
+func (c *Collector) DisableCookies() {
+ c.backend.Client.Jar = nil
+}
+
+// SetCookieJar overrides the previously set cookie jar
+func (c *Collector) SetCookieJar(j *cookiejar.Jar) {
+ c.backend.Client.Jar = j
+}
+
+// SetRequestTimeout overrides the default timeout (10 seconds) for this collector
+func (c *Collector) SetRequestTimeout(timeout time.Duration) {
+ c.backend.Client.Timeout = timeout
+}
+
+// SetStorage overrides the default in-memory storage.
+// Storage stores scraping related data like cookies and visited urls
+func (c *Collector) SetStorage(s storage.Storage) error {
+ if err := s.Init(); err != nil {
+ return err
+ }
+ c.store = s
+ c.backend.Client.Jar = createJar(s)
+ return nil
+}
+
+// SetProxy sets a proxy for the collector. This method overrides the previously
+// used http.Transport if the type of the transport is not http.RoundTripper.
+// The proxy type is determined by the URL scheme. "http"
+// and "socks5" are supported. If the scheme is empty,
+// "http" is assumed.
+func (c *Collector) SetProxy(proxyURL string) error {
+ proxyParsed, err := url.Parse(proxyURL)
+ if err != nil {
+ return err
+ }
+
+ c.SetProxyFunc(http.ProxyURL(proxyParsed))
+
+ return nil
+}
+
+// SetProxyFunc sets a custom proxy setter/switcher function.
+// See built-in ProxyFuncs for more details.
+// This method overrides the previously used http.Transport
+// if the type of the transport is not http.RoundTripper.
+// The proxy type is determined by the URL scheme. "http"
+// and "socks5" are supported. If the scheme is empty,
+// "http" is assumed.
+func (c *Collector) SetProxyFunc(p ProxyFunc) {
+ t, ok := c.backend.Client.Transport.(*http.Transport)
+ if c.backend.Client.Transport != nil && ok {
+ t.Proxy = p
+ } else {
+ c.backend.Client.Transport = &http.Transport{
+ Proxy: p,
+ }
+ }
+}
+
+func createEvent(eventType string, requestID, collectorID uint32, kvargs map[string]string) *debug.Event {
+ return &debug.Event{
+ CollectorID: collectorID,
+ RequestID: requestID,
+ Type: eventType,
+ Values: kvargs,
+ }
+}
+
+func (c *Collector) handleOnRequest(r *Request) {
+ if c.debugger != nil {
+ c.debugger.Event(createEvent("request", r.ID, c.ID, map[string]string{
+ "url": r.URL.String(),
+ }))
+ }
+ for _, f := range c.requestCallbacks {
+ f(r)
+ }
+}
+
+func (c *Collector) handleOnResponse(r *Response) {
+ if c.debugger != nil {
+ c.debugger.Event(createEvent("response", r.Request.ID, c.ID, map[string]string{
+ "url": r.Request.URL.String(),
+ "status": http.StatusText(r.StatusCode),
+ }))
+ }
+ for _, f := range c.responseCallbacks {
+ f(r)
+ }
+}
+
+func (c *Collector) handleOnHTML(resp *Response) error {
+ if len(c.htmlCallbacks) == 0 || !strings.Contains(strings.ToLower(resp.Headers.Get("Content-Type")), "html") {
+ return nil
+ }
+ doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(resp.Body))
+ if err != nil {
+ return err
+ }
+ if href, found := doc.Find("base[href]").Attr("href"); found {
+ resp.Request.baseURL, _ = url.Parse(href)
+ }
+ for _, cc := range c.htmlCallbacks {
+ i := 0
+ doc.Find(cc.Selector).Each(func(_ int, s *goquery.Selection) {
+ for _, n := range s.Nodes {
+ e := NewHTMLElementFromSelectionNode(resp, s, n, i)
+ i++
+ if c.debugger != nil {
+ c.debugger.Event(createEvent("html", resp.Request.ID, c.ID, map[string]string{
+ "selector": cc.Selector,
+ "url": resp.Request.URL.String(),
+ }))
+ }
+ cc.Function(e)
+ }
+ })
+ }
+ return nil
+}
+
+func (c *Collector) handleOnXML(resp *Response) error {
+ if len(c.xmlCallbacks) == 0 {
+ return nil
+ }
+ contentType := strings.ToLower(resp.Headers.Get("Content-Type"))
+ if !strings.Contains(contentType, "html") && !strings.Contains(contentType, "xml") {
+ return nil
+ }
+
+ if strings.Contains(contentType, "html") {
+ doc, err := htmlquery.Parse(bytes.NewBuffer(resp.Body))
+ if err != nil {
+ return err
+ }
+ if e := htmlquery.FindOne(doc, "//base/@href"); e != nil {
+ for _, a := range e.Attr {
+ if a.Key == "href" {
+ resp.Request.baseURL, _ = url.Parse(a.Val)
+ break
+ }
+ }
+ }
+
+ for _, cc := range c.xmlCallbacks {
+ htmlquery.FindEach(doc, cc.Query, func(i int, n *html.Node) {
+ e := NewXMLElementFromHTMLNode(resp, n)
+ if c.debugger != nil {
+ c.debugger.Event(createEvent("xml", resp.Request.ID, c.ID, map[string]string{
+ "selector": cc.Query,
+ "url": resp.Request.URL.String(),
+ }))
+ }
+ cc.Function(e)
+ })
+ }
+ } else if strings.Contains(contentType, "xml") {
+ doc, err := xmlquery.Parse(bytes.NewBuffer(resp.Body))
+ if err != nil {
+ return err
+ }
+
+ for _, cc := range c.xmlCallbacks {
+ xmlquery.FindEach(doc, cc.Query, func(i int, n *xmlquery.Node) {
+ e := NewXMLElementFromXMLNode(resp, n)
+ if c.debugger != nil {
+ c.debugger.Event(createEvent("xml", resp.Request.ID, c.ID, map[string]string{
+ "selector": cc.Query,
+ "url": resp.Request.URL.String(),
+ }))
+ }
+ cc.Function(e)
+ })
+ }
+ }
+ return nil
+}
+
+func (c *Collector) handleOnError(response *Response, err error, request *Request, ctx *Context) error {
+ if err == nil && (c.ParseHTTPErrorResponse || response.StatusCode < 203) {
+ return nil
+ }
+ if err == nil && response.StatusCode >= 203 {
+ err = errors.New(http.StatusText(response.StatusCode))
+ }
+ if response == nil {
+ response = &Response{
+ Request: request,
+ Ctx: ctx,
+ }
+ }
+ if c.debugger != nil {
+ c.debugger.Event(createEvent("error", request.ID, c.ID, map[string]string{
+ "url": request.URL.String(),
+ "status": http.StatusText(response.StatusCode),
+ }))
+ }
+ if response.Request == nil {
+ response.Request = request
+ }
+ if response.Ctx == nil {
+ response.Ctx = request.Ctx
+ }
+ for _, f := range c.errorCallbacks {
+ f(response, err)
+ }
+ return err
+}
+
+func (c *Collector) handleOnScraped(r *Response) {
+ if c.debugger != nil {
+ c.debugger.Event(createEvent("scraped", r.Request.ID, c.ID, map[string]string{
+ "url": r.Request.URL.String(),
+ }))
+ }
+ for _, f := range c.scrapedCallbacks {
+ f(r)
+ }
+}
+
+// Limit adds a new LimitRule to the collector
+func (c *Collector) Limit(rule *LimitRule) error {
+ return c.backend.Limit(rule)
+}
+
+// Limits adds new LimitRules to the collector
+func (c *Collector) Limits(rules []*LimitRule) error {
+ return c.backend.Limits(rules)
+}
+
+// SetCookies handles the receipt of the cookies in a reply for the given URL
+func (c *Collector) SetCookies(URL string, cookies []*http.Cookie) error {
+ if c.backend.Client.Jar == nil {
+ return ErrNoCookieJar
+ }
+ u, err := url.Parse(URL)
+ if err != nil {
+ return err
+ }
+ c.backend.Client.Jar.SetCookies(u, cookies)
+ return nil
+}
+
+// Cookies returns the cookies to send in a request for the given URL.
+func (c *Collector) Cookies(URL string) []*http.Cookie {
+ if c.backend.Client.Jar == nil {
+ return nil
+ }
+ u, err := url.Parse(URL)
+ if err != nil {
+ return nil
+ }
+ return c.backend.Client.Jar.Cookies(u)
+}
+
+// Clone creates an exact copy of a Collector without callbacks.
+// HTTP backend, robots.txt cache and cookie jar are shared
+// between collectors.
+func (c *Collector) Clone() *Collector {
+ return &Collector{
+ AllowedDomains: c.AllowedDomains,
+ AllowURLRevisit: c.AllowURLRevisit,
+ CacheDir: c.CacheDir,
+ DetectCharset: c.DetectCharset,
+ DisallowedDomains: c.DisallowedDomains,
+ ID: atomic.AddUint32(&collectorCounter, 1),
+ IgnoreRobotsTxt: c.IgnoreRobotsTxt,
+ MaxBodySize: c.MaxBodySize,
+ MaxDepth: c.MaxDepth,
+ DisallowedURLFilters: c.DisallowedURLFilters,
+ URLFilters: c.URLFilters,
+ ParseHTTPErrorResponse: c.ParseHTTPErrorResponse,
+ UserAgent: c.UserAgent,
+ store: c.store,
+ backend: c.backend,
+ debugger: c.debugger,
+ Async: c.Async,
+ RedirectHandler: c.RedirectHandler,
+ errorCallbacks: make([]ErrorCallback, 0, 8),
+ htmlCallbacks: make([]*htmlCallbackContainer, 0, 8),
+ xmlCallbacks: make([]*xmlCallbackContainer, 0, 8),
+ scrapedCallbacks: make([]ScrapedCallback, 0, 8),
+ lock: c.lock,
+ requestCallbacks: make([]RequestCallback, 0, 8),
+ responseCallbacks: make([]ResponseCallback, 0, 8),
+ robotsMap: c.robotsMap,
+ wg: &sync.WaitGroup{},
+ }
+}
+
+func (c *Collector) checkRedirectFunc() func(req *http.Request, via []*http.Request) error {
+ return func(req *http.Request, via []*http.Request) error {
+ if !c.isDomainAllowed(req.URL.Host) {
+ return fmt.Errorf("Not following redirect to %s because its not in AllowedDomains", req.URL.Host)
+ }
+
+ if c.RedirectHandler != nil {
+ return c.RedirectHandler(req, via)
+ }
+
+ // Honor golangs default of maximum of 10 redirects
+ if len(via) >= 10 {
+ return http.ErrUseLastResponse
+ }
+
+ lastRequest := via[len(via)-1]
+
+ // Copy the headers from last request
+ for hName, hValues := range lastRequest.Header {
+ for _, hValue := range hValues {
+ req.Header.Set(hName, hValue)
+ }
+ }
+
+ // If domain has changed, remove the Authorization-header if it exists
+ if req.URL.Host != lastRequest.URL.Host {
+ req.Header.Del("Authorization")
+ }
+
+ return nil
+ }
+}
+
+func (c *Collector) parseSettingsFromEnv() {
+ for _, e := range os.Environ() {
+ if !strings.HasPrefix(e, "COLLY_") {
+ continue
+ }
+ pair := strings.SplitN(e[6:], "=", 2)
+ if f, ok := envMap[pair[0]]; ok {
+ f(c, pair[1])
+ } else {
+ log.Println("Unknown environment variable:", pair[0])
+ }
+ }
+}
+
+// SanitizeFileName replaces dangerous characters in a string
+// so the return value can be used as a safe file name.
+func SanitizeFileName(fileName string) string {
+ ext := filepath.Ext(fileName)
+ cleanExt := sanitize.BaseName(ext)
+ if cleanExt == "" {
+ cleanExt = ".unknown"
+ }
+ return strings.Replace(fmt.Sprintf(
+ "%s.%s",
+ sanitize.BaseName(fileName[:len(fileName)-len(ext)]),
+ cleanExt[1:],
+ ), "-", "_", -1)
+}
+
+func createFormReader(data map[string]string) io.Reader {
+ form := url.Values{}
+ for k, v := range data {
+ form.Add(k, v)
+ }
+ return strings.NewReader(form.Encode())
+}
+
+func createMultipartReader(boundary string, data map[string][]byte) io.Reader {
+ dashBoundary := "--" + boundary
+
+ body := []byte{}
+ buffer := bytes.NewBuffer(body)
+
+ buffer.WriteString("Content-type: multipart/form-data; boundary=" + boundary + "\n\n")
+ for contentType, content := range data {
+ buffer.WriteString(dashBoundary + "\n")
+ buffer.WriteString("Content-Disposition: form-data; name=" + contentType + "\n")
+ buffer.WriteString(fmt.Sprintf("Content-Length: %d \n\n", len(content)))
+ buffer.Write(content)
+ buffer.WriteString("\n")
+ }
+ buffer.WriteString(dashBoundary + "--\n\n")
+ return buffer
+}
+
+// randomBoundary was borrowed from
+// github.com/golang/go/mime/multipart/writer.go#randomBoundary
+func randomBoundary() string {
+ var buf [30]byte
+ _, err := io.ReadFull(rand.Reader, buf[:])
+ if err != nil {
+ panic(err)
+ }
+ return fmt.Sprintf("%x", buf[:])
+}
+
+func isYesString(s string) bool {
+ switch strings.ToLower(s) {
+ case "1", "yes", "true", "y":
+ return true
+ }
+ return false
+}
+
+func createJar(s storage.Storage) http.CookieJar {
+ return &cookieJarSerializer{store: s, lock: &sync.RWMutex{}}
+}
+
+func (j *cookieJarSerializer) SetCookies(u *url.URL, cookies []*http.Cookie) {
+ j.lock.Lock()
+ defer j.lock.Unlock()
+ cookieStr := j.store.Cookies(u)
+
+ // Merge existing cookies, new cookies have precedence.
+ cnew := make([]*http.Cookie, len(cookies))
+ copy(cnew, cookies)
+ existing := storage.UnstringifyCookies(cookieStr)
+ for _, c := range existing {
+ if !storage.ContainsCookie(cnew, c.Name) {
+ cnew = append(cnew, c)
+ }
+ }
+ j.store.SetCookies(u, storage.StringifyCookies(cnew))
+}
+
+func (j *cookieJarSerializer) Cookies(u *url.URL) []*http.Cookie {
+ cookies := storage.UnstringifyCookies(j.store.Cookies(u))
+ // Filter.
+ now := time.Now()
+ cnew := make([]*http.Cookie, 0, len(cookies))
+ for _, c := range cookies {
+ // Drop expired cookies.
+ if c.RawExpires != "" && c.Expires.Before(now) {
+ continue
+ }
+ // Drop secure cookies if not over https.
+ if c.Secure && u.Scheme != "https" {
+ continue
+ }
+ cnew = append(cnew, c)
+ }
+ return cnew
+}
+
+func isMatchingFilter(fs []*regexp.Regexp, d []byte) bool {
+ for _, r := range fs {
+ if r.Match(d) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/vendor/github.com/gocolly/colly/colly_test.go b/vendor/github.com/gocolly/colly/colly_test.go
new file mode 100644
index 0000000..0fde6e7
--- /dev/null
+++ b/vendor/github.com/gocolly/colly/colly_test.go
@@ -0,0 +1,744 @@
+// Copyright 2018 Adam Tauber
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package colly
+
+import (
+ "bytes"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "reflect"
+ "regexp"
+ "strings"
+ "testing"
+
+ "github.com/PuerkitoBio/goquery"
+
+ "github.com/gocolly/colly/debug"
+)
+
+var serverIndexResponse = []byte("hello world\n")
+var robotsFile = `
+User-agent: *
+Allow: /allowed
+Disallow: /disallowed
+`
+
+func newTestServer() *httptest.Server {
+ mux := http.NewServeMux()
+
+ mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(200)
+ w.Write(serverIndexResponse)
+ })
+
+ mux.HandleFunc("/html", func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/html")
+ w.Write([]byte(`
+
+
+This is a test page
+This is a test paragraph
+ + + `)) + }) + + mux.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) { + if r.Method == "POST" { + w.Header().Set("Content-Type", "text/html") + w.Write([]byte(r.FormValue("name"))) + } + }) + + mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte(robotsFile)) + }) + + mux.HandleFunc("/allowed", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte("allowed")) + }) + + mux.HandleFunc("/disallowed", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte("disallowed")) + }) + + mux.Handle("/redirect", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/redirected/", http.StatusSeeOther) + + })) + + mux.Handle("/redirected/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `test`) + })) + + mux.HandleFunc("/set_cookie", func(w http.ResponseWriter, r *http.Request) { + c := &http.Cookie{Name: "test", Value: "testv", HttpOnly: false} + http.SetCookie(w, c) + w.WriteHeader(200) + w.Write([]byte("ok")) + }) + + mux.HandleFunc("/check_cookie", func(w http.ResponseWriter, r *http.Request) { + cs := r.Cookies() + if len(cs) != 1 || r.Cookies()[0].Value != "testv" { + w.WriteHeader(500) + w.Write([]byte("nok")) + return + } + w.WriteHeader(200) + w.Write([]byte("ok")) + }) + + mux.HandleFunc("/500", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(500) + w.Write([]byte("error
")) + }) + + mux.HandleFunc("/user_agent", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte(r.Header.Get("User-Agent"))) + }) + + mux.HandleFunc("/base", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.Write([]byte(` + + +a
b
c
+ This is a regular text paragraph. +
+`, ``}, + {"
Bold Not bold
\nAlso not bold.", "Bold Not bold\nAlso not bold."}, + {`FOO ZOO`, "FOO\rZOO"}, + {`">`, `alert("XSS")"`}, + {`![](https://example.com?d=https%3A%2F%2Fsome-evil-site.com%2Fimages%2Favatar.jpg%2f
+ <p>secret</p>`, `secret
+`},
+ {`<form action="https://some-evil-site.com"><button>Click</button><textarea name=)
-->
+`, `Click -- `},
+}
+
+func TestHTML(t *testing.T) {
+ for _, test := range htmlTests {
+ output := HTML(test.input)
+ if output != test.expected {
+ t.Fatalf(Format, test.input, test.expected, output)
+ }
+ }
+}
+
+var htmlTestsAllowing = []Test{
+ {`Span
`, `Span
`}, + {`test
`, `test
`}, + {`Some text
Some text
`}, + {`hello world`, `hello world`}, + {`textinside
too`, `textinside
too`}, + {`
`, `
`}, + {``, `
`}, + {"
Bold Not bold
\nAlso not bold.", "Bold Not bold
\nAlso not bold."}, + {"`FOO ZOO", "`FOO ZOO"}, + {`">`, `
+
+
+
+
+
+
+
+ PLAY
+
+
+
+
+
+ 
/storage/OJ/2012/highlight/ng2072242.jpg?type=highlight
/storage/OJ/2012/highlight/ng2068507.jpg?type=highlight
/storage/OJ/2012/highlight/ng2066787.jpg?type=highlight
/storage/OJ/2012/highlight/ng2060577.jpg?type=highlight
/storage/OJ/2012/highlight/ng2058164.jpg?type=highlight
/storage/OJ/2012/highlight/ng2036726.jpg?type=highlight
/storage/OJ/2012/highlight/ng2035701.jpg?type=highlight
/storage/OJ/2012/highlight/ng2053921.jpg?type=highlight
/storage/OJ/2012/highlight/ng2049530.jpg?type=highlight
/storage/OJ/2012/highlight/ng2048547.jpg?type=highlight
/storage/OJ/2012/highlight/ng2043204.jpg?type=highlight
/storage/OJ/2012/highlight/ng2034906.jpg?type=highlight
/storage/OJ/2012/highlight/ng2034306.jpg?type=highlight
/storage/OJ/2012/highlight/ng2032747.jpg?type=highlight
/storage/OJ/2012/highlight/ng2022893.jpg?type=highlight
/storage/OJ/2012/highlight/ng2021341.jpg?type=highlight
/storage/OJ/2012/highlight/ng2019226.jpg?type=highlight
/storage/OJ/2012/highlight/ng2012948.jpg?type=highlight
/storage/OJ/2012/highlight/ng2031453.jpg?type=highlight
/storage/OJ/2012/highlight/ng2020895.jpg?type=highlight
/storage/OJ/2012/highlight/ng2010500.jpg?type=highlight
/storage/OJ/2012/highlight/ng2037050.jpg?type=highlight
/storage/OJ/2012/highlight/ng2010639.jpg?type=highlight
/storage/OJ/2012/highlight/ng2003210.jpg?type=highlight
/storage/OJ/2012/highlight/ng1995334.jpg?type=highlight
/storage/OJ/2012/highlight/ng1983724.jpg?type=highlight
/storage/OJ/2012/highlight/ng1969028.jpg?type=highlight
/storage/OJ/2012/highlight/ng1972375.jpg?type=highlight
/storage/OJ/2012/highlight/ng1990881.jpg?type=highlight
/storage/OJ/2012/highlight/ng1964403.jpg?type=highlight
/storage/OJ/2012/highlight/ng1962609.jpg?type=highlight
/storage/OJ/2012/highlight/ng1954101.jpg?type=highlight
/storage/OJ/2012/highlight/ng1958449.jpg?type=highlight
/storage/OJ/2012/highlight/ng1958880.jpg?type=highlight
/storage/OJ/2012/highlight/ng1965697.jpg?type=highlight
+ 
+ 
O golo do agora jogador do Corunha abriu a vitória de Portugal sobre o Panamá no ensaio geral para o jogo do Luxemburgo, a 7 de setembro, a contar...
+ Comentar
+ 
GALERIA Lery Franco, de 30 anos, é eleita por muitos como a atleta mais sexy de todos os tempos. Confira 
Selecionador nacional referiu que até à expulsão de Gabriel Gómez o jogo foi interessante, em termos de treino da sua equipa.
+ Comentar
+ 
Capitão sublinha que até tentou pedir desculpa ao juiz no final do jogo
+ Comentar
+ 
PLAY Nascida na Venezuela, Marisol Lebre "trocou" Caracas por Vila Nova de Gaia e quer ajudar Portugal a melhorar o sector do Turismo. 
O Luxemburgo, que a equipa de Paulo Bento defronta a 7 de Setembro, até foi derrotada em casa pela Geórgia.
+ Comentar
+ 
Argentino Viola e chileno Diego Rubio também se encontram fora do lote escolhido pelo treinador Sá Pinto, mas ainda há vaga para mais um.
+ Comentar
+ 
A época vai começar, prepare-se da melhor maneira com o Guia da Liga 2012/13 d' O JOGO. Todas as estatísticas e todos os detalhes já à venda...
+ Comentar
+ 
Olhanense
Rio Ave
Benfica
Nacional
Paços de Ferreira
Gil Vicente
V. Guimarães
Beira-Mar
OlhanenseRio AveBenficaNacionalPaços de FerreiraGil VicenteV. GuimarãesBeira-Mar
Mano Menezes irá colocar Ramires de regresso ao lado direito no amigável pós-Jogos Olímpicos, voltando também à equipa David Luiz e Daniel Alves.
+ Comentar
+ 
Depois de uma época entre os craques, André Almeida foi relegado para a equipa B. Mas deseja rapidamente voltar a merecer a oportunidade de Jorge...
+ Comentar
+ 
Avançado oriundo do Racing Club de Avellaneda diz que já falou com Sá Pinto sobre o estilo de futebol em Portugal e entendeu-se muito bem. ...
+ Comentar
+ Dos quatro primeiros do último campeonato, qual está a realizar a melhor pré-época?
+
+
+
+
+
+ ![EXCLUSIVO J[###]Imogen Thomas, ex-amante de Giggs...](/storage/OJ/2012/small/ng2068102.jpg?type=small)
/storage/OJ/2012/highlight/ng2068101.jpg?type=highlight![EXCLUSIVO J[###]Sara Kostov, pai foi o 1º búlgaro...](/storage/OJ/2012/small/ng2062016.jpg?type=small)
/storage/OJ/2012/highlight/ng2062015.jpg?type=highlight![EXCLUSIVO J[###]Catarina Gouveia, capa da 140ª...](/storage/OJ/2012/small/ng2044764.jpg?type=small)
/storage/OJ/2012/highlight/ng2044763.jpg?type=highlight![EXCLUSIVO J[###]Pilar Prieto, capa da 198ª edição...](/storage/OJ/2012/small/ng2029109.jpg?type=small)
/storage/OJ/2012/highlight/ng2029108.jpg?type=highlight![EXCLUSIVO J[###]Silvana Kampmann, capa da 121ª...](/storage/OJ/2012/small/ng2053167.jpg?type=small)
/storage/OJ/2012/highlight/ng2053166.jpg?type=highlight![EXCLUSIVO J[###]Laura Figueiredo, capa da J conquistou...](/storage/OJ/2012/small/ng2018382.jpg?type=small)
/storage/OJ/2012/highlight/ng2018381.jpg?type=highlight![EXCLUSIVO J[###]Liliana Queiroz, capa da 55ª edição...](/storage/OJ/2012/small/ng2035233.jpg?type=small)
/storage/OJ/2012/highlight/ng2035232.jpg?type=highlight![EXCLUSIVO J[###]Ana Sofia, capa da 137ª edição...](/storage/OJ/2012/small/ng2004519.jpg?type=small)
/storage/OJ/2012/highlight/ng2004518.jpg?type=highlight![EXCLUSIVO J[###]Kim, a mais famosa das irmãs Kardashian](/storage/OJ/2012/small/ng1995284.jpg?type=small)
/storage/OJ/2012/highlight/ng1995283.jpg?type=highlight![EXCLUSIVO J[###]Margarida Neuparth, capa da 129ª...](/storage/OJ/2012/small/ng1989697.jpg?type=small)
/storage/OJ/2012/highlight/ng1989711.jpg?type=highlight![EXCLUSIVO J[###]Irina, a mulher que conquistou...](/storage/OJ/2012/small/ng1983912.jpg?type=small)
/storage/OJ/2012/highlight/ng1983911.jpg?type=highlight![EXCLUSIVO J[###]Estádio J - Portugal-Alemanha](/storage/OJ/2012/small/ng1978731.jpg?type=small)
/storage/OJ/2012/highlight/ng1978732.jpg?type=highlight![EXCLUSIVO J[###]Sara Dias, capa da 173ª edição...](/storage/OJ/2012/small/ng1974465.jpg?type=small)
/storage/OJ/2012/highlight/ng1974480.jpg?type=highlight![EXCLUSIVO J[###]Sara Santos, capa da J em duas...](/storage/OJ/2012/small/ng1958995.jpg?type=small)
/storage/OJ/2012/highlight/ng1960605.jpg?type=highlight![EXCLUSIVO J[###]Morgana Nogueira](/storage/OJ/2012/small/ng1958958.jpg?type=small)
/storage/OJ/2012/highlight/ng1960527.jpg?type=highlight![EXCLUSIVO J[###]Olívia Ortiz, a capa da J que é...](/storage/OJ/2012/small/ng1958883.jpg?type=small)
/storage/OJ/2012/highlight/ng1960551.jpg?type=highlight![EXCLUSIVO J[###]Rita Egídio, a primeira capa da...](/storage/OJ/2012/small/ng1967786.jpg?type=small)
/storage/OJ/2012/highlight/ng1967785.jpg?type=highlight![EXCLUSIVO J[###]Jaciara deu o pontapé de saída](/storage/OJ/2012/small/ng1962104.jpg?type=small)
/storage/OJ/2012/highlight/ng1962091.jpg?type=highlight| + | + | + |
+ + +
++ +
++ +
++ + +
++ +
++ +
++ +
+³¡¸¨®æ+ |
+ ¦^À³³Ì¼ö+ |
+
|
+
+
|
+
¡@¿W¦Û¶}©¹³¨³½¥xªº»´ä«O³¨²î¡u±ÒÂפG¸¹¡v¡A¬Q¤U¤È¥|ÂI¥b¥ª¥k¡A±aµÛ¤j³°¤¬PºX»P¤¤µØ¥Á°ê°êºX¦¨...
+![«j¦~»´¤H «i´±¬Û«H·Q¹³ªº¤O¶q ¤×§V´µ¡G§â³h½a°e¶i³Õª«À]](http://www.chinatimes.com/content-images/110101/C018160R.jpg)
¡@¿Õ¨©º¸©M¥¼ú±o¥D¿p¨uÀq¼w¡D¤×§V´µ¬Q¤é¦b¥xÆWºtÁ¿¡A¥Lªí¥Ü¡A¥ø·~¤£¥²µM¬°ÁÈ¿ú¦Ó¦s¦b¡A¥i¬°¤F¸Ñ...
+
¡m·s®Ñ²¤¶¡n¦Y¹ÚÃ~¬O©Çª«¶Ü¡H¸¥ÛÀܬO½Ö¡H¦Y¹ÚÃ~©M¥L¦³¤°»òÃö«Y¡H-->¦Y¹ÚÃ~±q¸¥ÛÀÜ¥X¥Í®É¡A´N...
+¼Ú¶Å·¼É½ÄÀ»º¥½w¡A°ê¤º°òª÷¥«³õ®ðª^ÂàÅs¡A®Ú¾Ú§ë«H§ëÅU¤½·|¤½¥¬³Ì·s²Îp¡A¦©°£¤µ¦~¤~¶Ò¶°¦¨¥ß©Î¤´³B³¬Âê´Áªº·s°òª÷¡A¤µ¦~¦Ü¤µ¥H¶Å¨é°òª÷³ÌºZ¾P¡A«e¤¦W¤À§O¬°²Ä¤@ª÷¥þ²y°ª¦¬¯q¡BÃv¨È¥þ²y°ª¦¬¯q...
+
®Ú¾Ú¥@¬É½Ã¥Í²Õ´¡]WHO¡^©ó¤µ¦~5¤ë¤½¥¬ªº³Ì·s¸ê®Æ«ü¥X¡A¥þ²y¨C3Ó¦¨¦~¤H·í¤¤¡A´N¦³1Ó¿©±w°ª...
+
¡@¦b¦U¬É°l¬d¡u´I¤Ö²]Å]¡v§õ©v·ç¡B¥~¬É¤]¤£Â_²q´ú¦³¨º¨Ç¤k©Ê¨ü®`¤§»Ú¡A³ü¶Ç´CºX¤U³ü©P¥Z©P¤T¥Zµn...
+¡@¬ü°ê¤@®a¨p¤H¾÷ºcCBB International¤é«e±À¥X¡u¤¤°ê¸gÀٽť֮ѡv¡A¦b¤£¨Ì¿à¤j³°©x¤è¸gÀټƾڪº±¡ªp¤U¡A±Ä¥Î»P¬ü°êÁp·Ç·|¬Û¦Pªº¬ã¨s»P½Õ¬d¤è¦¡¨Ó¤Ï¬M¤j³°ªº¸gÀÙ±¡ªp¡A¤º®e³zÅS...
+
¡@¥qªk°|±À°Êªº¡u¤H¥ÁÆ[¼f¨î¡v¡A½Ð¨ÓÃÀ¤H±i¶vÚ¬¥N¨¥¡A¬Q¤Ñ¤W¤ÈÁÙ¯S¦aÁ|¦æ«Å¾É¼v¤ù¤Î®ü³øµoªí·|¡F...
+
¡@¼Ú¬w¸gÀÙ¦M¾÷¤£¶ÈÅý¼Ú¤¸°ÏY¤z°ê®a³B©ó¯}²£Ãä½t¡A³Q¢°Çºò¿Ç¸yªº¼Ú¬w¦Ê©m¡AÀHµÛ¸gÀÙ°I°h¡A¡u¸g...
+¡@¥H§Þ³N±¾«Ó¡B¥«³õ¤ÎªA°È¾É¦Vªº±M·~¥b¾ÉÅé»sµ{³]³Æ¼t¡Ð»E©÷¬ì§Þ¡A¾Ö¦³PVD¡BCVD¤ÎPlasma Etcherµ¥3¤j®Ö¤ß³]³Æ§Þ³N¡A¤£ºÞ¬O¦bª¿¡B¤Æ¦Xª«¥b¾ÉÅé©Î¥D³Q°Ê¤¸¥óªº¥Í²£¹Lµ{¤¤¡A³£...
+¡@¤¤¥¡®ð¶H§½ªí¥Ü¡A¤µ¤Ñ¨ü¨ì»ä·¥~³ò¶³¨t¼vÅT¡A¤¤³¡¥H¥_©M©yÄõ¤È«á¦³¹p°}«B¡A¤U«Bªº®É¶¡¦³¥i¯à¤ñ¸ûªø¡C +¡@¥xªF¡B«í¬K¥b®q¡Bªá½¬¤Î«n³¡¦a°Ï¤£©w®É¦³µu¼È°}«B©Î¹p«B¡A¨ä¤¤¦b«í¬K¥b®q¤Î¥xªF¦a°Ï...
++ +
++ +
++ +
+ +
+¤è¤¯»È¤¦¥µ¡¼¥Ó¥¹
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Àµ¸á¤Þ¤Ç¤Î¤´Ãíʸ¤ÇºÇûÍâÆü¤Ë¤ªÆϤ±
+
+
+
+8·î16Æü(ÌÚ)
+
+
¥Í¥Ã¥È¥¹¡¼¥Ñ¡¼½ÐÁ°¡¦ÂðÇۥ쥹¥È¥é¥ó³ÚÅ·¥¤¡¼¥°¥ë¥¹
+
+
+
+¤ªµÞ¤®²¼¤µ¤¤¡ª8·î16Æü¤Þ¤Ç
+»Ë¾åºÇÂ硪ºÇÂç108,200¥Ý¥¤¥ó¥È
+¿¶¹þ¤àÅ٤˥ݥ¤¥ó¥È¤¶¤¯¤µ¤¯¡ª
+³ÚÅ·¤ÎÊݸ±5¼þǯ
+¥®¥Õ¥È¤ä´¹¶â¤â¤Ç¤¤ëÅŻҥޥ͡¼
+200±ß¤Ë¤Ä¤1¥Ý¥¤¥ó¥È¤¿¤Þ¤ë¡ª
+
¤ªÇ㤤ʪ¡¦¥Ý¥¤¥ó¥È¥Þ¥Í¡¼Î¹¹Ô¡¦¥¨¥ó¥¿¥áÊë¤é¤·¡¦¾ðÊó¥Ó¥¸¥Í¥¹³ÚÅ·¤Î¤ªÇ㤤ÆÀ¾ðÊó¡Ê̵ÎÁ¥á¥ë¥Þ¥¬¡Ë
+ +³ÚÅ·¤Ë¤ª¤±¤ë¾ðÊ󥻥¥å¥ê¥Æ¥£¤Î¼è¤êÁȤߤˤĤ¤¤Æ
+¢¨³ÚÅ·»Ô¾ì¤ÏSSL¤ËÂбþ¤·¤Æ¤¤¤ë¤Î¤Ç¡¢¥¯¥ì¥¸¥Ã¥È¥«¡¼¥ÉÈÖ¹æ¤Ï°Å¹æ²½¤·¤ÆÁ÷¿®¤µ¤ì¤Þ¤¹¡£
+
+
+
+
+
+
How to search for international shipping items
+¡Ê³¤³°ÈÎÇäÂоݾ¦ÉʤÎõ¤·Êý¤Ë¤Ä¤¤¤Æ¡Ë
UP
+ + +![[½ºÅ²] ³¯¾Æ¶ó ·¹µå¹öµå](http://c1img.nate.com/img/mall/items2/2012/07/1_angrybird_skin_6_s_c.gif)
![[¹Ì´Ï¹Ì] ¾Þ±×¸®! ·¹µå¹öµå](http://c1img.nate.com/img/myAvatar/2012/myAvatar_17005210_1.gif){kind=avatar}
РÎÅ¡¡Íø Ò³¡¡Ìù °É¡¡Öª µÀ¡¡MP3¡¡Í¼ Ƭ¡¡ÊÓ Æµ¡¡µØ ͼ
+°Ñ°Ù¶ÈÉèΪÖ÷Ò³°Ñ°Ù¶ÈÉèΪÖ÷Ò³°Ñ°Ù¶ÈÌí¼Óµ½×ÀÃæ
+¼ÓÈë°Ù¶ÈÍƹã | ËÑË÷·çÔÆ°ñ | ¹ØÓÚ°Ù¶È | About Baidu
©2012 Baidu ʹÓðٶÈÇ°±Ø¶Á ¾©ICPÖ¤030173ºÅ 
+
+
+
+ |
++ |
 |
++ |
 |
+ |
+ |
+ |
+ |
+ |
++ |
 |
+@ | ++ |
 |
+
 |
+
 |
+
 |
+
 |
+
 |
+
 |
+
 |
+
 |
+
 |
+
 |
+
 |
+
| + |
| + |
| + |  |
+|
 |
+ + |
+|
 |
+||
+ |
+  |
+|
 |
+||
+ |
+||
 |
+||
| @ | +  +  |
+
UTF-8 (UCS Transformation Format—8-bit[1]) is a variable-width encoding that can represent every character in the Unicode character set. It was designed for backward compatibility with ASCII and to avoid the complications of endianness and byte order marks in UTF-16 and UTF-32.
+UTF-8 has become the dominant character encoding for the World-Wide Web, accounting for more than half of all Web pages.[2][3][4] The Internet Engineering Task Force (IETF) requires all Internet protocols to identify the encoding used for character data, and the supported character encodings must include UTF-8.[5] The Internet Mail Consortium (IMC) recommends that all e-mail programs be able to display and create mail using UTF-8.[6] UTF-8 is also increasingly being used as the default character encoding in operating systems, programming languages, APIs, and software applications.[citation needed]
+UTF-8 encodes each of the 1,112,064[7] code points in the Unicode character set using one to four 8-bit bytes (termed "octets" in the Unicode Standard). Code points with lower numerical values (i.e. earlier code positions in the Unicode character set, which tend to occur more frequently in practice) are encoded using fewer bytes.[8] The first 128 characters of Unicode, which correspond one-to-one with ASCII, are encoded using a single octet with the same binary value as ASCII, making valid ASCII text valid UTF-8-encoded Unicode as well.
+The official IANA code for the UTF-8 character encoding is UTF-8.[9]
|
+
+
+Contents+
|
+
By early 1992 the search was on for a good byte-stream encoding of multi-byte character sets. The draft ISO 10646 standard contained a non-required annex called UTF-1 that provided a byte-stream encoding of its 32-bit code points. This encoding was not satisfactory on performance grounds, but did introduce the notion that bytes in the range of 0–127 continue representing the ASCII characters in UTF, thereby providing backward compatibility with ASCII.
+In July 1992, the X/Open committee XoJIG was looking for a better encoding. Dave Prosser of Unix System Laboratories submitted a proposal for one that had faster implementation characteristics and introduced the improvement that 7-bit ASCII characters would only represent themselves; all multibyte sequences would include only bytes where the high bit was set.
+In August 1992, this proposal was circulated by an IBM X/Open representative to interested parties. Ken Thompson of the Plan 9 operating system group at Bell Labs then made a crucial modification to the encoding to allow it to be self-synchronizing, meaning that it was not necessary to read from the beginning of the string to find code point boundaries. Thompson's design was outlined on September 2, 1992, on a placemat in a New Jersey diner with Rob Pike. The following days, Pike and Thompson implemented it and updated Plan 9 to use it throughout, and then communicated their success back to X/Open.[10]
+UTF-8 was first officially presented at the USENIX conference in San Diego, from January 25–29, 1993.
+In November 2003 UTF-8 was restricted by RFC 3629 to four bytes in order to match the constraints of the UTF-16 character encoding.
+The design of UTF‑8 is most easily seen in the table of the scheme as originally proposed by Dave Prosser and subsequently modified by Ken Thompson (the x's are replaced by the bits of the code point):
| Bits | +Last code point | +Byte 1 | +Byte 2 | +Byte 3 | +Byte 4 | +Byte 5 | +Byte 6 | +
|---|---|---|---|---|---|---|---|
| 7 | +U+007F | +0xxxxxxx |
+|||||
| 11 | +U+07FF | +110xxxxx |
+10xxxxxx |
+||||
| 16 | +U+FFFF | +1110xxxx |
+10xxxxxx |
+10xxxxxx |
+|||
| 21 | +U+1FFFFF | +11110xxx |
+10xxxxxx |
+10xxxxxx |
+10xxxxxx |
+||
| 26 | +U+3FFFFFF | +111110xx |
+10xxxxxx |
+10xxxxxx |
+10xxxxxx |
+10xxxxxx |
+|
| 31 | +U+7FFFFFFF | +1111110x |
+10xxxxxx |
+10xxxxxx |
+10xxxxxx |
+10xxxxxx |
+10xxxxxx |
+
The salient features of the above scheme are as follows:
+The first 128 characters (US-ASCII) need one byte. The next 1,920 characters need two bytes to encode. This covers the remainder of almost all Latin-derived alphabets, and also Greek, Cyrillic, Coptic, Armenian, Hebrew, Arabic, Syriac and TÄna alphabets, as well as Combining Diacritical Marks. Three bytes are needed for characters in the rest of the Basic Multilingual Plane (which contains virtually all characters in common use). Four bytes are needed for characters in the other planes of Unicode, which include less common CJK characters and various historic scripts and mathematical symbols.
+The original specification covered numbers up to 31 bits (the original limit of the Universal Character Set). In November 2003 UTF-8 was restricted by RFC 3629 to end at U+10FFFF, in order to match the constraints of the UTF-16 character encoding. This removed all 5- and 6-byte sequences, and about half of the 4-byte sequences.
Let us consider how to encode the Euro sign, €.
+
E quer tomar medidas rapidamente FIFA investiga incidente com...
Representante do médio reúne-se com a SAD Adrien volta à mesa de negociações
"O que se diz é puramente mediático" Messi diz que nunca fomentou...
Acidente em Pikes Peak. Assustador!PLAY
A presença de Nélson Oliveira
PLAY
Jorge Maia
José Manuel Ribeiro
Joel Neto
Carlos Flórido
+
+
+
+
diff --git a/vendor/github.com/saintfish/chardet/testdata/euc_jp.html b/vendor/github.com/saintfish/chardet/testdata/euc_jp.html
new file mode 100644
index 0000000..d20e232
--- /dev/null
+++ b/vendor/github.com/saintfish/chardet/testdata/euc_jp.html
@@ -0,0 +1,3584 @@
+
+
+
+
+
+
+
+ 
+ 
+ 
+ 
.gif)
(0).gif){kind=small}
![[¸Þ´ºÈ¿°ú] ·¹µå¹öµå _s](http://c1img.nate.com/img/mall/items/mainstyle/110917_ab4_m_c.gif)
³×ÀÌÆ®¿ÂUC
³×ÀÌÆ®¸ÞÀÏ
½ºÅ¸È¨ÇÇ
½ÎÀÌÇ÷¡±×
+@ƒIƒ“ƒ‰ƒCƒ“Ž–‹Æ‚ð‰‚ß‚ÄŽn‚ß‚½‚¢•û‚ÖAŠÈ’P‚ÅŒoÏ“I‚ȃEƒFƒuƒzƒXƒeƒBƒ“ƒO‚ð‚²’ñ‹Ÿ’v‚µ‚Ü‚·B20AC is binary 0010000010101100. The two leading zeros are added because, as the scheme table shows, a three-byte encoding needs exactly sixteen bits from the code point.1110...)11100010), leaving ...000010101100.10 and takes six bits of the code point (so 10000010, then 10101100).The three bytes 11100010 10000010 10101100 can be more concisely written in hexadecimal, as E2 82 AC.
The following table summarises this conversion, as well as others with different lengths in UTF-8. The colors indicate how bits from the code point are distributed among the UTF-8 bytes. Additional bits added by the UTF-8 encoding process are shown in black.
+| Character | +Binary code point | +Binary UTF-8 | +Hexadecimal UTF-8 | +|
|---|---|---|---|---|
| $ | +U+0024 |
+0100100 |
+00100100 |
+24 |
+
| ¢ | +U+00A2 |
+00010100010 |
+11000010 10100010 |
+C2 A2 |
+
| € | +U+20AC |
+0010000010101100 |
+11100010 10000010 10101100 |
+E2 82 AC |
+
| 𤢠| +U+24B62 |
+000100100101101100010 |
+11110000 10100100 10101101 10100010 |
+F0 A4 AD A2 |
+
The standard specifies that the correct encoding of a codepoint use only the minimum number of bytes required to hold the significant bits of the codepoint. Longer encodings are called overlong and are not valid UTF-8 representations of the codepoint. This rule maintains a one-to-one correspondence between codepoints and their valid encodings, so that there is a unique valid encoding for each codepoint. Allowing multiple encodings would make testing for string equality difficult to define.
+In principle, it would be possible to inflate the number of bytes in an encoding by padding the codepoint with leading 0s. To encode the Euro sign € from the above example in four bytes instead of three, it could be padded with leading 0s until it was 21 bits long—000000010000010101100. The leading byte prefix for a four-byte encoding is 11110, and so the complete, overlong encoding is 11110000 10000010 10000010 10101100 (or F0 82 82 AC in hexadecimal).
Although overlong encodings are forbidden in UTF-8, at least one derivative makes use of the form. Modified UTF-8 requires the Unicode codepoint U+0000 (the NUL character) to be encoded in the overlong form 11000000 10000000 (hex C0 80), rather than 00000000 (hex 00). This allows the byte 00 to be used as a string terminator.
| UTF-8 | +||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| + | _0 | +_1 | +_2 | +_3 | +_4 | +_5 | +_6 | +_7 | +_8 | +_9 | +_A | +_B | +_C | +_D | +_E | +_F | +
| +0_ + |
+NUL +0000 +0 |
+SOH +0001 +1 |
+STX +0002 +2 |
+ETX +0003 +3 |
+EOT +0004 +4 |
+ENQ +0005 +5 |
+ACK +0006 +6 |
+BEL +0007 +7 |
+BS +0008 +8 |
+HT +0009 +9 |
+LF +000A +10 |
+VT +000B +11 |
+FF +000C +12 |
+CR +000D +13 |
+SO +000E +14 |
+SI +000F +15 |
+
| +1_ + |
+DLE +0010 +16 |
+DC1 +0011 +17 |
+DC2 +0012 +18 |
+DC3 +0013 +19 |
+DC4 +0014 +20 |
+NAK +0015 +21 |
+SYN +0016 +22 |
+ETB +0017 +23 |
+CAN +0018 +24 |
+EM +0019 +25 |
+SUB +001A +26 |
+ESC +001B +27 |
+FS +001C +28 |
+GS +001D +29 |
+RS +001E +30 |
+US +001F +31 |
+
| +2_ + |
+SP +0020 +32 |
+! +0021 +33 |
+" +0022 +34 |
+# +0023 +35 |
+$ +0024 +36 |
+% +0025 +37 |
+& +0026 +38 |
+' +0027 +39 |
+( +0028 +40 |
+) +0029 +41 |
+* +002A +42 |
++ +002B +43 |
+, +002C +44 |
+- +002D +45 |
+. +002E +46 |
+/ +002F +47 |
+
| +3_ + |
+0 +0030 +48 |
+1 +0031 +49 |
+2 +0032 +50 |
+3 +0033 +51 |
+4 +0034 +52 |
+5 +0035 +53 |
+6 +0036 +54 |
+7 +0037 +55 |
+8 +0038 +56 |
+9 +0039 +57 |
+: +003A +58 |
+; +003B +59 |
+< +003C +60 |
+= +003D +61 |
+> +003E +62 |
+? +003F +63 |
+
| +4_ + |
+@ +0040 +64 |
+A +0041 +65 |
+B +0042 +66 |
+C +0043 +67 |
+D +0044 +68 |
+E +0045 +69 |
+F +0046 +70 |
+G +0047 +71 |
+H +0048 +72 |
+I +0049 +73 |
+J +004A +74 |
+K +004B +75 |
+L +004C +76 |
+M +004D +77 |
+N +004E +78 |
+O +004F +79 |
+
| +5_ + |
+P +0050 +80 |
+Q +0051 +81 |
+R +0052 +82 |
+S +0053 +83 |
+T +0054 +84 |
+U +0055 +85 |
+V +0056 +86 |
+W +0057 +87 |
+X +0058 +88 |
+Y +0059 +89 |
+Z +005A +90 |
+[ +005B +91 |
+\ +005C +92 |
+] +005D +93 |
+^ +005E +94 |
+_ +005F +95 |
+
| +6_ + |
+` +0060 +96 |
+a +0061 +97 |
+b +0062 +98 |
+c +0063 +99 |
+d +0064 +100 |
+e +0065 +101 |
+f +0066 +102 |
+g +0067 +103 |
+h +0068 +104 |
+i +0069 +105 |
+j +006A +106 |
+k +006B +107 |
+l +006C +108 |
+m +006D +109 |
+n +006E +110 |
+o +006F +111 |
+
| +7_ + |
+p +0070 +112 |
+q +0071 +113 |
+r +0072 +114 |
+s +0073 +115 |
+t +0074 +116 |
+u +0075 +117 |
+v +0076 +118 |
+w +0077 +119 |
+x +0078 +120 |
+y +0079 +121 |
+z +007A +122 |
+{ +007B +123 |
+| +007C +124 |
+} +007D +125 |
+~ +007E +126 |
+DEL +007F +127 |
+
| +8_ + |
+• ++00 +128 |
+• ++01 +129 |
+• ++02 +130 |
+• ++03 +131 |
+• ++04 +132 |
+• ++05 +133 |
+• ++06 +134 |
+• ++07 +135 |
+• ++08 +136 |
+• ++09 +137 |
+• ++0A +138 |
+• ++0B +139 |
+• ++0C +140 |
+• ++0D +141 |
+• ++0E +142 |
+• ++0F +143 |
+
| +9_ + |
+• ++10 +144 |
+• ++11 +145 |
+• ++12 +146 |
+• ++13 +147 |
+• ++14 +148 |
+• ++15 +149 |
+• ++16 +150 |
+• ++17 +151 |
+• ++18 +152 |
+• ++19 +153 |
+• ++1A +154 |
+• ++1B +155 |
+• ++1C +156 |
+• ++1D +157 |
+• ++1E +158 |
+• ++1F +159 |
+
| +A_ + |
+• ++20 +160 |
+• ++21 +161 |
+• ++22 +162 |
+• ++23 +163 |
+• ++24 +164 |
+• ++25 +165 |
+• ++26 +166 |
+• ++27 +167 |
+• ++28 +168 |
+• ++29 +169 |
+• ++2A +170 |
+• ++2B +171 |
+• ++2C +172 |
+• ++2D +173 |
+• ++2E +174 |
+• ++2F +175 |
+
| +B_ + |
+• ++30 +176 |
+• ++31 +177 |
+• ++32 +178 |
+• ++33 +179 |
+• ++34 +180 |
+• ++35 +181 |
+• ++36 +182 |
+• ++37 +183 |
+• ++38 +184 |
+• ++39 +185 |
+• ++3A +186 |
+• ++3B +187 |
+• ++3C +188 |
+• ++3D +189 |
+• ++3E +190 |
+• ++3F +191 |
+
| +2-byte +C_ + |
+2-byte +inval +(0000) +192 |
+2-byte +inval +(0040) +193 |
+Latin-1 +0080 +194 |
+Latin-1 +00C0 +195 |
+Latin +Ext-A +0100 +196 |
+Latin +Ext-A +0140 +197 |
+Latin +Ext-B +0180 +198 |
+Latin +Ext-B +01C0 +199 |
+Latin +Ext-B +0200 +200 |
+IPA +0240 +201 |
+IPA +0280 +202 |
+Spaci +Modif +02C0 +203 |
+Combi +Diacr +0300 +204 |
+Combi +Diacr +0340 +205 |
+Greek +0380 +206 |
+Greek +03C0 +207 |
+
| +2-byte +D_ + |
+Cyril +0400 +208 |
+Cyril +0440 +209 |
+Cyril +0480 +210 |
+Cyril +04C0 +211 |
+Cyril +0500 +212 |
+Armen +0540 +213 |
+Hebrew +0580 +214 |
+Hebrew +05C0 +215 |
+Arabic +0600 +216 |
+Arabic +0640 +217 |
+Arabic +0680 +218 |
+Arabic +06C0 +219 |
+Syriac +0700 +220 |
+Arabic +0740 +221 |
+Thaana +0780 +222 |
+N'Ko +07C0 +223 |
+
| +3-byte +E_ + |
+Indic +0800* +224 |
+Misc. +1000 +225 |
+Symbol +2000 +226 |
+Kana +CJK +3000 +227 |
+CJK +4000 +228 |
+CJK +5000 +229 |
+CJK +6000 +230 |
+CJK +7000 +231 |
+CJK +8000 +232 |
+CJK +9000 +233 |
+Asian +A000 +234 |
+Hangul +B000 +235 |
+Hangul +C000 +236 |
+Hangul +Surr +D000 +237 |
+Priv Use +E000 +238 |
+Forms +F000 +239 |
+
| +4-byte +F_ + |
+Ancient +Sym,CJK +10000* +240 |
+unall +40000 +241 |
+unall +80000 +242 |
+Tags +Priv +C0000 +243 |
+Priv +Use +100000 +244 |
+4-byte +inval +140000 +245 |
+4-byte +inval +180000 +246 |
+4-byte +inval +1C0000 +247 |
+5-byte +inval +200000* +248 |
+5-byte +inval +1000000 +249 |
+5-byte +inval +2000000 +250 |
+5-byte +inval +3000000 +251 |
+6-byte +inval +4000000* +252 |
+6-byte +inval +40000000 +253 |
++ +254 |
++ +255 |
+
Legend: Yellow cells are control characters, blue cells are punctuation, purple cells are digits and green cells are ASCII letters.
+Orange cells with a large dot are continuation bytes. The hexadecimal number shown after a "+" plus sign is the value of the 6 bits they add.
+White cells are the start bytes for a sequence of multiple bytes, the length shown at the left edge of the row. The text shows the Unicode blocks encoded by sequences starting with this byte, and the hexadecimal code point shown in the cell is the lowest character value encoded using that start byte. When a start byte could form both overlong and valid encodings, the lowest non-overlong-encoded codepoint is shown, marked by an asterisk "*".
+Red cells must never appear in a valid UTF-8 sequence. The first two (C0 and C1) could only be used for overlong encoding of basic ASCII characters. The remaining red cells indicate start bytes of sequences that could only encode numbers larger than the 0x10FFFF limit of Unicode. The byte 244 (hex 0xF4) could also encode some values greater than 0x10FFFF; such a sequence is also invalid.
+Not all sequences of bytes are valid UTF-8. A UTF-8 decoder should be prepared for:
+Many earlier decoders would happily try to decode these. Carefully crafted invalid UTF-8 could make them either skip or create ASCII characters such as NUL, slash, or quotes. Invalid UTF-8 has been used to bypass security validations in high profile products including Microsoft's IIS web server[11] and Apache's Tomcat servlet container.[12]
+RFC 3629 states "Implementations of the decoding algorithm MUST protect against decoding invalid sequences."[13] The Unicode Standard requires decoders to "...treat any ill-formed code unit sequence as an error condition. This guarantees that it will neither interpret nor emit an ill-formed code unit sequence."
+Many UTF-8 decoders throw exceptions on encountering errors,[14] since such errors suggest the input is not a UTF-8 string at all. This can turn what would otherwise be harmless errors (producing a message such as "no such file") into a denial of service bug. For instance, Python 3.0 would exit immediately if the command line or environment variables contained invalid UTF-8,[15] so it was impossible for any Python program to detect and recover from such an error.
+An increasingly popular option is to detect errors with a separate API, and for converters to translate the first byte to a replacement and continue parsing with the next byte. Popular replacements are:
+Replacing errors is "lossy": more than one UTF-8 string converts to the same Unicode result. Therefore the original UTF-8 should be stored, and translation should only be used when displaying the text to the user.
+According to the UTF-8 definition (RFC 3629) the high and low surrogate halves used by UTF-16 (U+D800 through U+DFFF) are not legal Unicode values, and the UTF-8 encoding of them is an invalid byte sequence and thus should be treated as described above.
+Whether an actual application should do this with surrogate halves is debatable.[who?] Allowing them allows lossless storage of invalid UTF-16, and allows CESU encoding (described below) to be decoded. There are other code points that are far more important to detect and reject, such as the reversed-BOM U+FFFE, or the C1 controls, caused by improper conversion of CP1252 text or double-encoding of UTF-8. These are invalid in HTML.
+The official name is "UTF-8". All letters are upper-case, and the name is hyphenated. This spelling is used in all the documents relating to the encoding.
+Alternatively, the name "utf-8" may be used by all standards conforming to the Internet Assigned Numbers Authority (IANA) list (which include CSS, HTML, XML, and HTTP headers),[16] as the declaration is case insensitive.[17]
+Other descriptions that omit the hyphen or replace it with a space, such as "utf8" or "UTF 8", are not accepted as correct by the governing standards.[18] Despite this, most agents such as browsers can understand them, and so standards intended to describe existing practice (such as HTML5) may effectively require their recognition.
+MySQL omits the hyphen in the following query:
++SET NAMES 'utf8' ++
The following implementations show slight differences from the UTF-8 specification. They are incompatible with the UTF-8 specification.
+Many programs added UTF-8 conversions for UCS-2 data and did not alter this UTF-8 conversion when UCS-2 was replaced with the surrogate-pair using UTF-16. In such programs each half of a UTF-16 surrogate pair is encoded as its own 3-byte UTF-8 encoding, resulting in 6-byte sequences rather than 4 bytes for characters outside the Basic Multilingual Plane. Oracle and MySQL databases use this, as well as Java and Tcl as described below, and probably many Windows programs where the programmers were unaware of the complexities of UTF-16. Although this non-optimal encoding is generally not deliberate, a supposed benefit is that it preserves UTF-16 binary sorting order when CESU-8 is binary sorted.
+In Modified UTF-8,[19] the null character (U+0000) is encoded as 0xC0,0x80; this is not valid UTF-8[20] because it is not the shortest possible representation. Modified UTF-8 strings never contain any actual null bytes but can contain all Unicode code points including U+0000,[21] which allows such strings (with a null byte appended) to be processed by traditional null-terminated string functions.
+All known Modified UTF-8 implementations also treat the surrogate pairs as in CESU-8.
+In normal usage, the Java programming language supports standard UTF-8 when reading and writing strings through InputStreamReader and OutputStreamWriter. However it uses Modified UTF-8 for object serialization,[22] for the Java Native Interface,[23] and for embedding constant strings in class files.[24] Tcl also uses the same modified UTF-8[25] as Java for internal representation of Unicode data, but uses strict CESU-8 for external data.
Extending the accepted input pattern from 6 bytes to 7 bytes would allow over 70 billion code points to be encoded;[26] however, this would require an initial byte value of 0xFE to be accepted as a 7-byte sequence indicator (see under Advantages in section "Compared to single-byte encodings").
+Many Windows programs (including Windows Notepad) add the bytes 0xEF, 0xBB, 0xBF at the start of any document saved as UTF-8. This is the UTF-8 encoding of the Unicode byte order mark (BOM), and is commonly referred to as a UTF-8 BOM, even though it is not relevant to byte order. The BOM can also appear if another encoding with a BOM is translated to UTF-8 without stripping it. Older text editors may display the BOM as "" at the start of the document.
+The Unicode Standard neither requires nor recommends the use of the BOM for UTF-8.[27] The presence of the UTF-8 BOM may cause interoperability problems with existing software that could otherwise handle UTF-8; for example:
+If compatibility with existing programs is not important, the BOM could be used to identify UTF-8 encoding. Because checking if text is valid UTF-8 is very reliable (the majority of random byte sequences are not valid UTF-8) such use should not be necessary. Programs that insert information at the start of a file will break this identification (one example is offline browsers that add the originating URL to the start of the file).
+Unofficially, UTF-8-BOM or UTF-8-NOBOM are sometimes used to refer to text files which contain or lack a BOM. In Japan especially, "UTF-8 encoding without BOM" is sometimes called "UTF-8N".[citation needed]
+ |
+This section needs additional citations for verification. (October 2009) | +
There are several current definitions of UTF-8 in various standards documents:
+They supersede the definitions given in the following obsolete works:
+They are all the same in their general mechanics, with the main differences being on issues such as allowed range of code point values and safe handling of invalid input.
+
+
|
+|||||||||||||||||||||||||||||||||||||||||||||||||||||
+
|
+|||||||||||||||||||||||||||||||||||||||||||||||
+
|
+|||||||||||||||||
+
|
+||||||||||||||
UTF-8 (UCS Transformation Format—8-bit[1]) is a variable-width encoding that can represent every character in the Unicode character set. It was designed for backward compatibility with ASCII and to avoid the complications of endianness and byte order marks in UTF-16 and UTF-32.
+UTF-8 has become the dominant character encoding for the World-Wide Web, accounting for more than half of all Web pages.[2][3][4] The Internet Engineering Task Force (IETF) requires all Internet protocols to identify the encoding used for character data, and the supported character encodings must include UTF-8.[5] The Internet Mail Consortium (IMC) recommends that all e-mail programs be able to display and create mail using UTF-8.[6] UTF-8 is also increasingly being used as the default character encoding in operating systems, programming languages, APIs, and software applications.[citation needed]
+UTF-8 encodes each of the 1,112,064[7] code points in the Unicode character set using one to four 8-bit bytes (termed "octets" in the Unicode Standard). Code points with lower numerical values (i.e. earlier code positions in the Unicode character set, which tend to occur more frequently in practice) are encoded using fewer bytes.[8] The first 128 characters of Unicode, which correspond one-to-one with ASCII, are encoded using a single octet with the same binary value as ASCII, making valid ASCII text valid UTF-8-encoded Unicode as well.
+The official IANA code for the UTF-8 character encoding is UTF-8.[9]
|
+
+
+Contents+
|
+
By early 1992 the search was on for a good byte-stream encoding of multi-byte character sets. The draft ISO 10646 standard contained a non-required annex called UTF-1 that provided a byte-stream encoding of its 32-bit code points. This encoding was not satisfactory on performance grounds, but did introduce the notion that bytes in the range of 0–127 continue representing the ASCII characters in UTF, thereby providing backward compatibility with ASCII.
+In July 1992, the X/Open committee XoJIG was looking for a better encoding. Dave Prosser of Unix System Laboratories submitted a proposal for one that had faster implementation characteristics and introduced the improvement that 7-bit ASCII characters would only represent themselves; all multibyte sequences would include only bytes where the high bit was set.
+In August 1992, this proposal was circulated by an IBM X/Open representative to interested parties. Ken Thompson of the Plan 9 operating system group at Bell Labs then made a crucial modification to the encoding to allow it to be self-synchronizing, meaning that it was not necessary to read from the beginning of the string to find code point boundaries. Thompson's design was outlined on September 2, 1992, on a placemat in a New Jersey diner with Rob Pike. The following days, Pike and Thompson implemented it and updated Plan 9 to use it throughout, and then communicated their success back to X/Open.[10]
+UTF-8 was first officially presented at the USENIX conference in San Diego, from January 25–29, 1993.
+In November 2003 UTF-8 was restricted by RFC 3629 to four bytes in order to match the constraints of the UTF-16 character encoding.
+The design of UTF‑8 is most easily seen in the table of the scheme as originally proposed by Dave Prosser and subsequently modified by Ken Thompson (the x's are replaced by the bits of the code point):
| Bits | +Last code point | +Byte 1 | +Byte 2 | +Byte 3 | +Byte 4 | +Byte 5 | +Byte 6 | +
|---|---|---|---|---|---|---|---|
| 7 | +U+007F | +0xxxxxxx |
+|||||
| 11 | +U+07FF | +110xxxxx |
+10xxxxxx |
+||||
| 16 | +U+FFFF | +1110xxxx |
+10xxxxxx |
+10xxxxxx |
+|||
| 21 | +U+1FFFFF | +11110xxx |
+10xxxxxx |
+10xxxxxx |
+10xxxxxx |
+||
| 26 | +U+3FFFFFF | +111110xx |
+10xxxxxx |
+10xxxxxx |
+10xxxxxx |
+10xxxxxx |
+|
| 31 | +U+7FFFFFFF | +1111110x |
+10xxxxxx |
+10xxxxxx |
+10xxxxxx |
+10xxxxxx |
+10xxxxxx |
+
The salient features of the above scheme are as follows:
+The first 128 characters (US-ASCII) need one byte. The next 1,920 characters need two bytes to encode. This covers the remainder of almost all Latin-derived alphabets, and also Greek, Cyrillic, Coptic, Armenian, Hebrew, Arabic, Syriac and TÄna alphabets, as well as Combining Diacritical Marks. Three bytes are needed for characters in the rest of the Basic Multilingual Plane (which contains virtually all characters in common use). Four bytes are needed for characters in the other planes of Unicode, which include less common CJK characters and various historic scripts and mathematical symbols.
+The original specification covered numbers up to 31 bits (the original limit of the Universal Character Set). In November 2003 UTF-8 was restricted by RFC 3629 to end at U+10FFFF, in order to match the constraints of the UTF-16 character encoding. This removed all 5- and 6-byte sequences, and about half of the 4-byte sequences.
Let us consider how to encode the Euro sign, €.
+
20AC is binary 0010000010101100. The two leading zeros are added because, as the scheme table shows, a three-byte encoding needs exactly sixteen bits from the code point.1110...)11100010), leaving ...000010101100.10 and takes six bits of the code point (so 10000010, then 10101100).The three bytes 11100010 10000010 10101100 can be more concisely written in hexadecimal, as E2 82 AC.
The following table summarises this conversion, as well as others with different lengths in UTF-8. The colors indicate how bits from the code point are distributed among the UTF-8 bytes. Additional bits added by the UTF-8 encoding process are shown in black.
+| Character | +Binary code point | +Binary UTF-8 | +Hexadecimal UTF-8 | +|
|---|---|---|---|---|
| $ | +U+0024 |
+0100100 |
+00100100 |
+24 |
+
| ¢ | +U+00A2 |
+00010100010 |
+11000010 10100010 |
+C2 A2 |
+
| € | +U+20AC |
+0010000010101100 |
+11100010 10000010 10101100 |
+E2 82 AC |
+
| 𤢠| +U+24B62 |
+000100100101101100010 |
+11110000 10100100 10101101 10100010 |
+F0 A4 AD A2 |
+
The standard specifies that the correct encoding of a codepoint use only the minimum number of bytes required to hold the significant bits of the codepoint. Longer encodings are called overlong and are not valid UTF-8 representations of the codepoint. This rule maintains a one-to-one correspondence between codepoints and their valid encodings, so that there is a unique valid encoding for each codepoint. Allowing multiple encodings would make testing for string equality difficult to define.
+In principle, it would be possible to inflate the number of bytes in an encoding by padding the codepoint with leading 0s. To encode the Euro sign € from the above example in four bytes instead of three, it could be padded with leading 0s until it was 21 bits long—000000010000010101100. The leading byte prefix for a four-byte encoding is 11110, and so the complete, overlong encoding is 11110000 10000010 10000010 10101100 (or F0 82 82 AC in hexadecimal).
Although overlong encodings are forbidden in UTF-8, at least one derivative makes use of the form. Modified UTF-8 requires the Unicode codepoint U+0000 (the NUL character) to be encoded in the overlong form 11000000 10000000 (hex C0 80), rather than 00000000 (hex 00). This allows the byte 00 to be used as a string terminator.
| UTF-8 | +||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| + | _0 | +_1 | +_2 | +_3 | +_4 | +_5 | +_6 | +_7 | +_8 | +_9 | +_A | +_B | +_C | +_D | +_E | +_F | +
| +0_ + |
+NUL +0000 +0 |
+SOH +0001 +1 |
+STX +0002 +2 |
+ETX +0003 +3 |
+EOT +0004 +4 |
+ENQ +0005 +5 |
+ACK +0006 +6 |
+BEL +0007 +7 |
+BS +0008 +8 |
+HT +0009 +9 |
+LF +000A +10 |
+VT +000B +11 |
+FF +000C +12 |
+CR +000D +13 |
+SO +000E +14 |
+SI +000F +15 |
+
| +1_ + |
+DLE +0010 +16 |
+DC1 +0011 +17 |
+DC2 +0012 +18 |
+DC3 +0013 +19 |
+DC4 +0014 +20 |
+NAK +0015 +21 |
+SYN +0016 +22 |
+ETB +0017 +23 |
+CAN +0018 +24 |
+EM +0019 +25 |
+SUB +001A +26 |
+ESC +001B +27 |
+FS +001C +28 |
+GS +001D +29 |
+RS +001E +30 |
+US +001F +31 |
+
| +2_ + |
+SP +0020 +32 |
+! +0021 +33 |
+" +0022 +34 |
+# +0023 +35 |
+$ +0024 +36 |
+% +0025 +37 |
+& +0026 +38 |
+' +0027 +39 |
+( +0028 +40 |
+) +0029 +41 |
+* +002A +42 |
++ +002B +43 |
+, +002C +44 |
+- +002D +45 |
+. +002E +46 |
+/ +002F +47 |
+
| +3_ + |
+0 +0030 +48 |
+1 +0031 +49 |
+2 +0032 +50 |
+3 +0033 +51 |
+4 +0034 +52 |
+5 +0035 +53 |
+6 +0036 +54 |
+7 +0037 +55 |
+8 +0038 +56 |
+9 +0039 +57 |
+: +003A +58 |
+; +003B +59 |
+< +003C +60 |
+= +003D +61 |
+> +003E +62 |
+? +003F +63 |
+
| +4_ + |
+@ +0040 +64 |
+A +0041 +65 |
+B +0042 +66 |
+C +0043 +67 |
+D +0044 +68 |
+E +0045 +69 |
+F +0046 +70 |
+G +0047 +71 |
+H +0048 +72 |
+I +0049 +73 |
+J +004A +74 |
+K +004B +75 |
+L +004C +76 |
+M +004D +77 |
+N +004E +78 |
+O +004F +79 |
+
| +5_ + |
+P +0050 +80 |
+Q +0051 +81 |
+R +0052 +82 |
+S +0053 +83 |
+T +0054 +84 |
+U +0055 +85 |
+V +0056 +86 |
+W +0057 +87 |
+X +0058 +88 |
+Y +0059 +89 |
+Z +005A +90 |
+[ +005B +91 |
+\ +005C +92 |
+] +005D +93 |
+^ +005E +94 |
+_ +005F +95 |
+
| +6_ + |
+` +0060 +96 |
+a +0061 +97 |
+b +0062 +98 |
+c +0063 +99 |
+d +0064 +100 |
+e +0065 +101 |
+f +0066 +102 |
+g +0067 +103 |
+h +0068 +104 |
+i +0069 +105 |
+j +006A +106 |
+k +006B +107 |
+l +006C +108 |
+m +006D +109 |
+n +006E +110 |
+o +006F +111 |
+
| +7_ + |
+p +0070 +112 |
+q +0071 +113 |
+r +0072 +114 |
+s +0073 +115 |
+t +0074 +116 |
+u +0075 +117 |
+v +0076 +118 |
+w +0077 +119 |
+x +0078 +120 |
+y +0079 +121 |
+z +007A +122 |
+{ +007B +123 |
+| +007C +124 |
+} +007D +125 |
+~ +007E +126 |
+DEL +007F +127 |
+
| +8_ + |
+• ++00 +128 |
+• ++01 +129 |
+• ++02 +130 |
+• ++03 +131 |
+• ++04 +132 |
+• ++05 +133 |
+• ++06 +134 |
+• ++07 +135 |
+• ++08 +136 |
+• ++09 +137 |
+• ++0A +138 |
+• ++0B +139 |
+• ++0C +140 |
+• ++0D +141 |
+• ++0E +142 |
+• ++0F +143 |
+
| +9_ + |
+• ++10 +144 |
+• ++11 +145 |
+• ++12 +146 |
+• ++13 +147 |
+• ++14 +148 |
+• ++15 +149 |
+• ++16 +150 |
+• ++17 +151 |
+• ++18 +152 |
+• ++19 +153 |
+• ++1A +154 |
+• ++1B +155 |
+• ++1C +156 |
+• ++1D +157 |
+• ++1E +158 |
+• ++1F +159 |
+
| +A_ + |
+• ++20 +160 |
+• ++21 +161 |
+• ++22 +162 |
+• ++23 +163 |
+• ++24 +164 |
+• ++25 +165 |
+• ++26 +166 |
+• ++27 +167 |
+• ++28 +168 |
+• ++29 +169 |
+• ++2A +170 |
+• ++2B +171 |
+• ++2C +172 |
+• ++2D +173 |
+• ++2E +174 |
+• ++2F +175 |
+
| +B_ + |
+• ++30 +176 |
+• ++31 +177 |
+• ++32 +178 |
+• ++33 +179 |
+• ++34 +180 |
+• ++35 +181 |
+• ++36 +182 |
+• ++37 +183 |
+• ++38 +184 |
+• ++39 +185 |
+• ++3A +186 |
+• ++3B +187 |
+• ++3C +188 |
+• ++3D +189 |
+• ++3E +190 |
+• ++3F +191 |
+
| +2-byte +C_ + |
+2-byte +inval +(0000) +192 |
+2-byte +inval +(0040) +193 |
+Latin-1 +0080 +194 |
+Latin-1 +00C0 +195 |
+Latin +Ext-A +0100 +196 |
+Latin +Ext-A +0140 +197 |
+Latin +Ext-B +0180 +198 |
+Latin +Ext-B +01C0 +199 |
+Latin +Ext-B +0200 +200 |
+IPA +0240 +201 |
+IPA +0280 +202 |
+Spaci +Modif +02C0 +203 |
+Combi +Diacr +0300 +204 |
+Combi +Diacr +0340 +205 |
+Greek +0380 +206 |
+Greek +03C0 +207 |
+
| +2-byte +D_ + |
+Cyril +0400 +208 |
+Cyril +0440 +209 |
+Cyril +0480 +210 |
+Cyril +04C0 +211 |
+Cyril +0500 +212 |
+Armen +0540 +213 |
+Hebrew +0580 +214 |
+Hebrew +05C0 +215 |
+Arabic +0600 +216 |
+Arabic +0640 +217 |
+Arabic +0680 +218 |
+Arabic +06C0 +219 |
+Syriac +0700 +220 |
+Arabic +0740 +221 |
+Thaana +0780 +222 |
+N'Ko +07C0 +223 |
+
| +3-byte +E_ + |
+Indic +0800* +224 |
+Misc. +1000 +225 |
+Symbol +2000 +226 |
+Kana +CJK +3000 +227 |
+CJK +4000 +228 |
+CJK +5000 +229 |
+CJK +6000 +230 |
+CJK +7000 +231 |
+CJK +8000 +232 |
+CJK +9000 +233 |
+Asian +A000 +234 |
+Hangul +B000 +235 |
+Hangul +C000 +236 |
+Hangul +Surr +D000 +237 |
+Priv Use +E000 +238 |
+Forms +F000 +239 |
+
| +4-byte +F_ + |
+Ancient +Sym,CJK +10000* +240 |
+unall +40000 +241 |
+unall +80000 +242 |
+Tags +Priv +C0000 +243 |
+Priv +Use +100000 +244 |
+4-byte +inval +140000 +245 |
+4-byte +inval +180000 +246 |
+4-byte +inval +1C0000 +247 |
+5-byte +inval +200000* +248 |
+5-byte +inval +1000000 +249 |
+5-byte +inval +2000000 +250 |
+5-byte +inval +3000000 +251 |
+6-byte +inval +4000000* +252 |
+6-byte +inval +40000000 +253 |
++ +254 |
++ +255 |
+
Legend: Yellow cells are control characters, blue cells are punctuation, purple cells are digits and green cells are ASCII letters.
+Orange cells with a large dot are continuation bytes. The hexadecimal number shown after a "+" plus sign is the value of the 6 bits they add.
+White cells are the start bytes for a sequence of multiple bytes, the length shown at the left edge of the row. The text shows the Unicode blocks encoded by sequences starting with this byte, and the hexadecimal code point shown in the cell is the lowest character value encoded using that start byte. When a start byte could form both overlong and valid encodings, the lowest non-overlong-encoded codepoint is shown, marked by an asterisk "*".
+Red cells must never appear in a valid UTF-8 sequence. The first two (C0 and C1) could only be used for overlong encoding of basic ASCII characters. The remaining red cells indicate start bytes of sequences that could only encode numbers larger than the 0x10FFFF limit of Unicode. The byte 244 (hex 0xF4) could also encode some values greater than 0x10FFFF; such a sequence is also invalid.
+Not all sequences of bytes are valid UTF-8. A UTF-8 decoder should be prepared for:
+Many earlier decoders would happily try to decode these. Carefully crafted invalid UTF-8 could make them either skip or create ASCII characters such as NUL, slash, or quotes. Invalid UTF-8 has been used to bypass security validations in high profile products including Microsoft's IIS web server[11] and Apache's Tomcat servlet container.[12]
+RFC 3629 states "Implementations of the decoding algorithm MUST protect against decoding invalid sequences."[13] The Unicode Standard requires decoders to "...treat any ill-formed code unit sequence as an error condition. This guarantees that it will neither interpret nor emit an ill-formed code unit sequence."
+Many UTF-8 decoders throw exceptions on encountering errors,[14] since such errors suggest the input is not a UTF-8 string at all. This can turn what would otherwise be harmless errors (producing a message such as "no such file") into a denial of service bug. For instance, Python 3.0 would exit immediately if the command line or environment variables contained invalid UTF-8,[15] so it was impossible for any Python program to detect and recover from such an error.
+An increasingly popular option is to detect errors with a separate API, and for converters to translate the first byte to a replacement and continue parsing with the next byte. Popular replacements are:
+Replacing errors is "lossy": more than one UTF-8 string converts to the same Unicode result. Therefore the original UTF-8 should be stored, and translation should only be used when displaying the text to the user.
+According to the UTF-8 definition (RFC 3629) the high and low surrogate halves used by UTF-16 (U+D800 through U+DFFF) are not legal Unicode values, and the UTF-8 encoding of them is an invalid byte sequence and thus should be treated as described above.
+Whether an actual application should do this with surrogate halves is debatable.[who?] Allowing them allows lossless storage of invalid UTF-16, and allows CESU encoding (described below) to be decoded. There are other code points that are far more important to detect and reject, such as the reversed-BOM U+FFFE, or the C1 controls, caused by improper conversion of CP1252 text or double-encoding of UTF-8. These are invalid in HTML.
+The official name is "UTF-8". All letters are upper-case, and the name is hyphenated. This spelling is used in all the documents relating to the encoding.
+Alternatively, the name "utf-8" may be used by all standards conforming to the Internet Assigned Numbers Authority (IANA) list (which include CSS, HTML, XML, and HTTP headers),[16] as the declaration is case insensitive.[17]
+Other descriptions that omit the hyphen or replace it with a space, such as "utf8" or "UTF 8", are not accepted as correct by the governing standards.[18] Despite this, most agents such as browsers can understand them, and so standards intended to describe existing practice (such as HTML5) may effectively require their recognition.
+MySQL omits the hyphen in the following query:
++SET NAMES 'utf8' ++
The following implementations show slight differences from the UTF-8 specification. They are incompatible with the UTF-8 specification.
+Many programs added UTF-8 conversions for UCS-2 data and did not alter this UTF-8 conversion when UCS-2 was replaced with the surrogate-pair using UTF-16. In such programs each half of a UTF-16 surrogate pair is encoded as its own 3-byte UTF-8 encoding, resulting in 6-byte sequences rather than 4 bytes for characters outside the Basic Multilingual Plane. Oracle and MySQL databases use this, as well as Java and Tcl as described below, and probably many Windows programs where the programmers were unaware of the complexities of UTF-16. Although this non-optimal encoding is generally not deliberate, a supposed benefit is that it preserves UTF-16 binary sorting order when CESU-8 is binary sorted.
+In Modified UTF-8,[19] the null character (U+0000) is encoded as 0xC0,0x80; this is not valid UTF-8[20] because it is not the shortest possible representation. Modified UTF-8 strings never contain any actual null bytes but can contain all Unicode code points including U+0000,[21] which allows such strings (with a null byte appended) to be processed by traditional null-terminated string functions.
+All known Modified UTF-8 implementations also treat the surrogate pairs as in CESU-8.
+In normal usage, the Java programming language supports standard UTF-8 when reading and writing strings through InputStreamReader and OutputStreamWriter. However it uses Modified UTF-8 for object serialization,[22] for the Java Native Interface,[23] and for embedding constant strings in class files.[24] Tcl also uses the same modified UTF-8[25] as Java for internal representation of Unicode data, but uses strict CESU-8 for external data.
Extending the accepted input pattern from 6 bytes to 7 bytes would allow over 70 billion code points to be encoded;[26] however, this would require an initial byte value of 0xFE to be accepted as a 7-byte sequence indicator (see under Advantages in section "Compared to single-byte encodings").
+Many Windows programs (including Windows Notepad) add the bytes 0xEF, 0xBB, 0xBF at the start of any document saved as UTF-8. This is the UTF-8 encoding of the Unicode byte order mark (BOM), and is commonly referred to as a UTF-8 BOM, even though it is not relevant to byte order. The BOM can also appear if another encoding with a BOM is translated to UTF-8 without stripping it. Older text editors may display the BOM as "" at the start of the document.
+The Unicode Standard neither requires nor recommends the use of the BOM for UTF-8.[27] The presence of the UTF-8 BOM may cause interoperability problems with existing software that could otherwise handle UTF-8; for example:
+If compatibility with existing programs is not important, the BOM could be used to identify UTF-8 encoding. Because checking if text is valid UTF-8 is very reliable (the majority of random byte sequences are not valid UTF-8) such use should not be necessary. Programs that insert information at the start of a file will break this identification (one example is offline browsers that add the originating URL to the start of the file).
+Unofficially, UTF-8-BOM or UTF-8-NOBOM are sometimes used to refer to text files which contain or lack a BOM. In Japan especially, "UTF-8 encoding without BOM" is sometimes called "UTF-8N".[citation needed]
+ |
+This section needs additional citations for verification. (October 2009) | +
There are several current definitions of UTF-8 in various standards documents:
+They supersede the definitions given in the following obsolete works:
+They are all the same in their general mechanics, with the main differences being on issues such as allowed range of code point values and safe handling of invalid input.
+
+
|
+|||||||||||||||||||||||||||||||||||||||||||||||||||||
+
|
+|||||||||||||||||||||||||||||||||||||||||||||||
+
|
+|||||||||||||||||
+
|
+||||||||||||||
Hello world! This is valid HTML but invalid robots.txt.`
+
+func TestHtmlInstead(t *testing.T) {
+ r, err := FromString(robotsTextJustHTML)
+ if err != nil {
+ // According to Google spec, invalid robots.txt file
+ // must be parsed silently.
+ t.Fatal(err.Error())
+ }
+ group := r.FindGroup("SuperBot")
+ if group == nil {
+ t.Fatal("Group must not be nil.")
+ }
+ if !group.Test("/") {
+ t.Fatal("Must allow by default.")
+ }
+}
+
+// http://perche.vanityfair.it/robots.txt on Sat, 13 Sep 2014 23:00:29 GMT
+const robotsTextVanityfair = "\xef\xbb\xbfUser-agent: *\nDisallow: */oroscopo-di-oggi/*"
+
+func TestWildcardPrefix(t *testing.T) {
+ r, err := FromString(robotsTextVanityfair)
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+ if !r.TestAgent("/foo/bar", "WildcardBot") {
+ t.Fatal("Must allow /foo/bar")
+ }
+ if r.TestAgent("/oroscopo-di-oggi/bar", "WildcardBot") {
+ t.Fatal("Must not allow /oroscopo-di-oggi/bar")
+ }
+ if r.TestAgent("/foo/oroscopo-di-oggi/bar", "WildcardBot") {
+ t.Fatal("Must not allow /foo/oroscopo-di-oggi/bar")
+ }
+}
+
+func TestGrouping(t *testing.T) {
+ const robotsCaseGrouping = `user-agent: a
+user-agent: b
+disallow: /a
+disallow: /b
+
+user-agent: ignore
+Disallow: /separator
+
+user-agent: b
+user-agent: c
+disallow: /b
+disallow: /c`
+
+ if r, e := FromString(robotsCaseGrouping); e != nil {
+ t.Fatal(e)
+ } else {
+ ExpectDisallow(t, r, "/a", "a")
+ ExpectDisallow(t, r, "/b", "a")
+ ExpectAllow(t, r, "/c", "a")
+
+ ExpectDisallow(t, r, "/a", "b")
+ ExpectDisallow(t, r, "/b", "b")
+ ExpectDisallow(t, r, "/c", "b")
+
+ ExpectAllow(t, r, "/a", "c")
+ ExpectDisallow(t, r, "/b", "c")
+ ExpectDisallow(t, r, "/c", "c")
+ }
+}
+
+func BenchmarkParseFromString001(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ FromString(robotsText001)
+ b.SetBytes(int64(len(robotsText001)))
+ }
+}
+
+func BenchmarkParseFromString002(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ FromString(robotsText002)
+ b.SetBytes(int64(len(robotsText002)))
+ }
+}
+
+func BenchmarkParseFromStatus401(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ FromStatusAndString(401, "")
+ }
+}
diff --git a/vendor/github.com/temoto/robotstxt/scanner.go b/vendor/github.com/temoto/robotstxt/scanner.go
new file mode 100644
index 0000000..1577637
--- /dev/null
+++ b/vendor/github.com/temoto/robotstxt/scanner.go
@@ -0,0 +1,205 @@
+package robotstxt
+
+import (
+ "bytes"
+ "fmt"
+ "go/token"
+ "io"
+ "os"
+ "unicode/utf8"
+)
+
+type byteScanner struct {
+ ErrorCount int
+ Quiet bool
+
+ buf []byte
+ pos token.Position
+ lastChunk bool
+ ch rune
+ keyTokenFound bool
+}
+
+var WhitespaceChars = []rune{' ', '\t', '\v'}
+
+func newByteScanner(srcname string, quiet bool) *byteScanner {
+ return &byteScanner{
+ Quiet: quiet,
+ ch: -1,
+ pos: token.Position{Filename: srcname},
+ }
+}
+
+func (s *byteScanner) Feed(input []byte, end bool) error {
+ s.buf = input
+ s.pos.Offset = 0
+ s.pos.Line = 1
+ s.pos.Column = 1
+ s.lastChunk = end
+
+ // Read first char into look-ahead buffer `s.ch`.
+ if err := s.nextChar(); err != nil {
+ return err
+ }
+
+ // Skip UTF-8 byte order mark
+ if s.ch == 65279 {
+ s.nextChar()
+ s.pos.Column = 1
+ }
+
+ return nil
+}
+
+func (s *byteScanner) GetPosition() token.Position {
+ return s.pos
+}
+
+func (s *byteScanner) Scan() (string, error) {
+ //println("--- Scan(). Offset / len(s.buf): ", s.pos.Offset, len(s.buf))
+
+ for {
+ // Note Offset > len, not >=, so we can Scan last character.
+ if s.lastChunk && s.pos.Offset > len(s.buf) {
+ return "", io.EOF
+ }
+
+ s.skipSpace()
+
+ if s.ch == -1 {
+ return "", io.EOF
+ }
+
+ // EOL
+ if s.isEol() {
+ s.keyTokenFound = false
+ // skip subsequent newline chars
+ for s.ch != -1 && s.isEol() {
+ s.nextChar()
+ }
+ // emit newline as separate token
+ return "\n", nil
+ }
+
+ // skip comments
+ if s.ch == '#' {
+ s.keyTokenFound = false
+ s.skipUntilEol()
+ // s.state = "start"
+ if s.ch == -1 {
+ return "", io.EOF
+ }
+ // emit newline as separate token
+ return "\n", nil
+ }
+
+ // else we found something
+ break
+ }
+
+ /*
+ if s.state == "start" {
+ s.state = "key"
+ }
+ */
+
+ var tok bytes.Buffer
+ tok.WriteRune(s.ch)
+ s.nextChar()
+ for s.ch != -1 && !s.isSpace() && !s.isEol() {
+ // Do not consider ":" to be a token separator if a first key token
+ // has already been found on this line (avoid cutting an absolute URL
+ // after the "http:")
+ if s.ch == ':' && !s.keyTokenFound {
+ // s.state = "pre-value"
+ s.nextChar()
+ s.keyTokenFound = true
+ break
+ }
+
+ tok.WriteRune(s.ch)
+ s.nextChar()
+ }
+ return tok.String(), nil
+}
+
+func (s *byteScanner) ScanAll() ([]string, error) {
+ var results []string
+ for {
+ t, err := s.Scan()
+ if t != "" {
+ results = append(results, t)
+ }
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return results, err
+ }
+ }
+ return results, nil
+}
+
+func (s *byteScanner) error(pos token.Position, msg string) {
+ s.ErrorCount++
+ if !s.Quiet {
+ fmt.Fprintf(os.Stderr, "robotstxt from %s: %s\n", pos.String(), msg)
+ }
+}
+
+func (s *byteScanner) isEol() bool {
+ return s.ch == '\n' || s.ch == '\r'
+}
+
+func (s *byteScanner) isSpace() bool {
+ for _, r := range WhitespaceChars {
+ if s.ch == r {
+ return true
+ }
+ }
+ return false
+}
+
+func (s *byteScanner) skipSpace() {
+ //println("--- string(ch): ", s.ch, ".")
+ for s.ch != -1 && s.isSpace() {
+ s.nextChar()
+ }
+}
+
+func (s *byteScanner) skipUntilEol() {
+ //println("--- string(ch): ", s.ch, ".")
+ for s.ch != -1 && !s.isEol() {
+ s.nextChar()
+ }
+ // skip subsequent newline chars
+ for s.ch != -1 && s.isEol() {
+ s.nextChar()
+ }
+}
+
+// Reads next Unicode char.
+func (s *byteScanner) nextChar() error {
+ //println("--- nextChar(). Offset / len(s.buf): ", s.pos.Offset, len(s.buf))
+
+ if s.pos.Offset >= len(s.buf) {
+ s.ch = -1
+ return io.EOF
+ }
+ s.pos.Column++
+ if s.ch == '\n' {
+ s.pos.Line++
+ s.pos.Column = 1
+ }
+ r, w := rune(s.buf[s.pos.Offset]), 1
+ if r >= 0x80 {
+ r, w = utf8.DecodeRune(s.buf[s.pos.Offset:])
+ if r == utf8.RuneError && w == 1 {
+ s.error(s.pos, "illegal UTF-8 encoding")
+ }
+ }
+ s.pos.Column++
+ s.pos.Offset += w
+ s.ch = r
+ return nil
+}
diff --git a/vendor/github.com/temoto/robotstxt/scanner_test.go b/vendor/github.com/temoto/robotstxt/scanner_test.go
new file mode 100644
index 0000000..26104c5
--- /dev/null
+++ b/vendor/github.com/temoto/robotstxt/scanner_test.go
@@ -0,0 +1,96 @@
+package robotstxt
+
+import (
+ // "os"
+ "strconv"
+ "testing"
+)
+
+func TestScan001(t *testing.T) {
+ sc := newByteScanner("test-001", false)
+ if _, err := sc.Scan(); err == nil {
+ t.Fatal("Empty ByteScanner should fail on Scan.")
+ }
+}
+
+func TestScan002(t *testing.T) {
+ sc := newByteScanner("test-002", false)
+ sc.Feed([]byte("foo"), true)
+ _, err := sc.Scan()
+ //print("---", tok, err)
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+}
+
+func TestScan004(t *testing.T) {
+ sc := newByteScanner("test-004", false)
+ sc.Feed([]byte("\u2010"), true)
+ _, err := sc.Scan()
+ //println("---", tok, err)
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+}
+
+func TestScan005(t *testing.T) {
+ sc := newByteScanner("test-005", true)
+ sc.Feed([]byte("\xd9\xd9"), true)
+ _, err := sc.Scan()
+ //println("---", tok, err)
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+ if sc.ErrorCount != 2 {
+ t.Fatal("Expecting ErrorCount be exactly 2.")
+ }
+}
+
+func TestScan006(t *testing.T) {
+ sc := newByteScanner("test-006", false)
+ s := "# comment \r\nSomething: Somewhere\r\n"
+ sc.Feed([]byte(s), true)
+ tokens, err := sc.ScanAll()
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+ //println("--- len(tokens):", len(tokens))
+ if len(tokens) != 4 {
+ t.Fatal("Expecting exactly 4 tokens.")
+ }
+ if tokens[0] != "\n" || tokens[1] != "Something" || tokens[2] != "Somewhere" || tokens[3] != "\n" {
+ t.Fatal("Wrong tokens read:", strconv.Quote(tokens[0]), strconv.Quote(tokens[1]), strconv.Quote(tokens[2]), strconv.Quote(tokens[3]))
+ }
+}
+
+func TestScan007(t *testing.T) {
+ sc := newByteScanner("test-007", false)
+ s := "# comment \r\n# more comments\n\nDisallow:\r"
+ sc.Feed([]byte(s), true)
+ tokens, err := sc.ScanAll()
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+ //println("--- len(tokens):", len(tokens))
+ if len(tokens) != 4 {
+ t.Fatal("Expecting exactly 4 tokens.")
+ }
+ if tokens[0] != "\n" || tokens[1] != "\n" || tokens[2] != "Disallow" || tokens[3] != "\n" {
+ t.Fatal("Wrong tokens read:", strconv.Quote(tokens[0]), strconv.Quote(tokens[1]), strconv.Quote(tokens[2]), strconv.Quote(tokens[3]))
+ }
+}
+
+func TestScanUnicode8BOM(t *testing.T) {
+ sc := newByteScanner("test-bom", false)
+ sc.Feed([]byte(robotsTextVanityfair), true)
+ tokens, err := sc.ScanAll()
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+ if len(tokens) == 0 {
+ t.Fatal("Read zero tokens.")
+ }
+ if tokens[0] != "User-agent" {
+ t.Fatal("Expecting first token: User-agent")
+ }
+}
diff --git a/vendor/golang.org/x/net/LICENSE b/vendor/golang.org/x/net/LICENSE
new file mode 100644
index 0000000..6a66aea
--- /dev/null
+++ b/vendor/golang.org/x/net/LICENSE
@@ -0,0 +1,27 @@
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/vendor/golang.org/x/net/PATENTS b/vendor/golang.org/x/net/PATENTS
new file mode 100644
index 0000000..7330990
--- /dev/null
+++ b/vendor/golang.org/x/net/PATENTS
@@ -0,0 +1,22 @@
+Additional IP Rights Grant (Patents)
+
+"This implementation" means the copyrightable works distributed by
+Google as part of the Go project.
+
+Google hereby grants to You a perpetual, worldwide, non-exclusive,
+no-charge, royalty-free, irrevocable (except as stated in this section)
+patent license to make, have made, use, offer to sell, sell, import,
+transfer and otherwise run, modify and propagate the contents of this
+implementation of Go, where such license applies only to those patent
+claims, both currently owned or controlled by Google and acquired in
+the future, licensable by Google that are necessarily infringed by this
+implementation of Go. This grant does not include claims that would be
+infringed only as a consequence of further modification of this
+implementation. If you or your agent or exclusive licensee institute or
+order or agree to the institution of patent litigation against any
+entity (including a cross-claim or counterclaim in a lawsuit) alleging
+that this implementation of Go or any code incorporated within this
+implementation of Go constitutes direct or contributory patent
+infringement, or inducement of patent infringement, then any patent
+rights granted to you under this License for this implementation of Go
+shall terminate as of the date such litigation is filed.
diff --git a/vendor/golang.org/x/net/context/context.go b/vendor/golang.org/x/net/context/context.go
new file mode 100644
index 0000000..a3c021d
--- /dev/null
+++ b/vendor/golang.org/x/net/context/context.go
@@ -0,0 +1,56 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package context defines the Context type, which carries deadlines,
+// cancelation signals, and other request-scoped values across API boundaries
+// and between processes.
+// As of Go 1.7 this package is available in the standard library under the
+// name context. https://golang.org/pkg/context.
+//
+// Incoming requests to a server should create a Context, and outgoing calls to
+// servers should accept a Context. The chain of function calls between must
+// propagate the Context, optionally replacing it with a modified copy created
+// using WithDeadline, WithTimeout, WithCancel, or WithValue.
+//
+// Programs that use Contexts should follow these rules to keep interfaces
+// consistent across packages and enable static analysis tools to check context
+// propagation:
+//
+// Do not store Contexts inside a struct type; instead, pass a Context
+// explicitly to each function that needs it. The Context should be the first
+// parameter, typically named ctx:
+//
+// func DoSomething(ctx context.Context, arg Arg) error {
+// // ... use ctx ...
+// }
+//
+// Do not pass a nil Context, even if a function permits it. Pass context.TODO
+// if you are unsure about which Context to use.
+//
+// Use context Values only for request-scoped data that transits processes and
+// APIs, not for passing optional parameters to functions.
+//
+// The same Context may be passed to functions running in different goroutines;
+// Contexts are safe for simultaneous use by multiple goroutines.
+//
+// See http://blog.golang.org/context for example code for a server that uses
+// Contexts.
+package context // import "golang.org/x/net/context"
+
+// Background returns a non-nil, empty Context. It is never canceled, has no
+// values, and has no deadline. It is typically used by the main function,
+// initialization, and tests, and as the top-level Context for incoming
+// requests.
+func Background() Context {
+ return background
+}
+
+// TODO returns a non-nil, empty Context. Code should use context.TODO when
+// it's unclear which Context to use or it is not yet available (because the
+// surrounding function has not yet been extended to accept a Context
+// parameter). TODO is recognized by static analysis tools that determine
+// whether Contexts are propagated correctly in a program.
+func TODO() Context {
+ return todo
+}
diff --git a/vendor/golang.org/x/net/context/context_test.go b/vendor/golang.org/x/net/context/context_test.go
new file mode 100644
index 0000000..6284413
--- /dev/null
+++ b/vendor/golang.org/x/net/context/context_test.go
@@ -0,0 +1,583 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !go1.7
+
+package context
+
+import (
+ "fmt"
+ "math/rand"
+ "runtime"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+// otherContext is a Context that's not one of the types defined in context.go.
+// This lets us test code paths that differ based on the underlying type of the
+// Context.
+type otherContext struct {
+ Context
+}
+
+func TestBackground(t *testing.T) {
+ c := Background()
+ if c == nil {
+ t.Fatalf("Background returned nil")
+ }
+ select {
+ case x := <-c.Done():
+ t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
+ default:
+ }
+ if got, want := fmt.Sprint(c), "context.Background"; got != want {
+ t.Errorf("Background().String() = %q want %q", got, want)
+ }
+}
+
+func TestTODO(t *testing.T) {
+ c := TODO()
+ if c == nil {
+ t.Fatalf("TODO returned nil")
+ }
+ select {
+ case x := <-c.Done():
+ t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
+ default:
+ }
+ if got, want := fmt.Sprint(c), "context.TODO"; got != want {
+ t.Errorf("TODO().String() = %q want %q", got, want)
+ }
+}
+
+func TestWithCancel(t *testing.T) {
+ c1, cancel := WithCancel(Background())
+
+ if got, want := fmt.Sprint(c1), "context.Background.WithCancel"; got != want {
+ t.Errorf("c1.String() = %q want %q", got, want)
+ }
+
+ o := otherContext{c1}
+ c2, _ := WithCancel(o)
+ contexts := []Context{c1, o, c2}
+
+ for i, c := range contexts {
+ if d := c.Done(); d == nil {
+ t.Errorf("c[%d].Done() == %v want non-nil", i, d)
+ }
+ if e := c.Err(); e != nil {
+ t.Errorf("c[%d].Err() == %v want nil", i, e)
+ }
+
+ select {
+ case x := <-c.Done():
+ t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
+ default:
+ }
+ }
+
+ cancel()
+ time.Sleep(100 * time.Millisecond) // let cancelation propagate
+
+ for i, c := range contexts {
+ select {
+ case <-c.Done():
+ default:
+ t.Errorf("<-c[%d].Done() blocked, but shouldn't have", i)
+ }
+ if e := c.Err(); e != Canceled {
+ t.Errorf("c[%d].Err() == %v want %v", i, e, Canceled)
+ }
+ }
+}
+
+func TestParentFinishesChild(t *testing.T) {
+ // Context tree:
+ // parent -> cancelChild
+ // parent -> valueChild -> timerChild
+ parent, cancel := WithCancel(Background())
+ cancelChild, stop := WithCancel(parent)
+ defer stop()
+ valueChild := WithValue(parent, "key", "value")
+ timerChild, stop := WithTimeout(valueChild, 10000*time.Hour)
+ defer stop()
+
+ select {
+ case x := <-parent.Done():
+ t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
+ case x := <-cancelChild.Done():
+ t.Errorf("<-cancelChild.Done() == %v want nothing (it should block)", x)
+ case x := <-timerChild.Done():
+ t.Errorf("<-timerChild.Done() == %v want nothing (it should block)", x)
+ case x := <-valueChild.Done():
+ t.Errorf("<-valueChild.Done() == %v want nothing (it should block)", x)
+ default:
+ }
+
+ // The parent's children should contain the two cancelable children.
+ pc := parent.(*cancelCtx)
+ cc := cancelChild.(*cancelCtx)
+ tc := timerChild.(*timerCtx)
+ pc.mu.Lock()
+ if len(pc.children) != 2 || !pc.children[cc] || !pc.children[tc] {
+ t.Errorf("bad linkage: pc.children = %v, want %v and %v",
+ pc.children, cc, tc)
+ }
+ pc.mu.Unlock()
+
+ if p, ok := parentCancelCtx(cc.Context); !ok || p != pc {
+ t.Errorf("bad linkage: parentCancelCtx(cancelChild.Context) = %v, %v want %v, true", p, ok, pc)
+ }
+ if p, ok := parentCancelCtx(tc.Context); !ok || p != pc {
+ t.Errorf("bad linkage: parentCancelCtx(timerChild.Context) = %v, %v want %v, true", p, ok, pc)
+ }
+
+ cancel()
+
+ pc.mu.Lock()
+ if len(pc.children) != 0 {
+ t.Errorf("pc.cancel didn't clear pc.children = %v", pc.children)
+ }
+ pc.mu.Unlock()
+
+ // parent and children should all be finished.
+ check := func(ctx Context, name string) {
+ select {
+ case <-ctx.Done():
+ default:
+ t.Errorf("<-%s.Done() blocked, but shouldn't have", name)
+ }
+ if e := ctx.Err(); e != Canceled {
+ t.Errorf("%s.Err() == %v want %v", name, e, Canceled)
+ }
+ }
+ check(parent, "parent")
+ check(cancelChild, "cancelChild")
+ check(valueChild, "valueChild")
+ check(timerChild, "timerChild")
+
+ // WithCancel should return a canceled context on a canceled parent.
+ precanceledChild := WithValue(parent, "key", "value")
+ select {
+ case <-precanceledChild.Done():
+ default:
+ t.Errorf("<-precanceledChild.Done() blocked, but shouldn't have")
+ }
+ if e := precanceledChild.Err(); e != Canceled {
+ t.Errorf("precanceledChild.Err() == %v want %v", e, Canceled)
+ }
+}
+
+func TestChildFinishesFirst(t *testing.T) {
+ cancelable, stop := WithCancel(Background())
+ defer stop()
+ for _, parent := range []Context{Background(), cancelable} {
+ child, cancel := WithCancel(parent)
+
+ select {
+ case x := <-parent.Done():
+ t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
+ case x := <-child.Done():
+ t.Errorf("<-child.Done() == %v want nothing (it should block)", x)
+ default:
+ }
+
+ cc := child.(*cancelCtx)
+ pc, pcok := parent.(*cancelCtx) // pcok == false when parent == Background()
+ if p, ok := parentCancelCtx(cc.Context); ok != pcok || (ok && pc != p) {
+ t.Errorf("bad linkage: parentCancelCtx(cc.Context) = %v, %v want %v, %v", p, ok, pc, pcok)
+ }
+
+ if pcok {
+ pc.mu.Lock()
+ if len(pc.children) != 1 || !pc.children[cc] {
+ t.Errorf("bad linkage: pc.children = %v, cc = %v", pc.children, cc)
+ }
+ pc.mu.Unlock()
+ }
+
+ cancel()
+
+ if pcok {
+ pc.mu.Lock()
+ if len(pc.children) != 0 {
+ t.Errorf("child's cancel didn't remove self from pc.children = %v", pc.children)
+ }
+ pc.mu.Unlock()
+ }
+
+ // child should be finished.
+ select {
+ case <-child.Done():
+ default:
+ t.Errorf("<-child.Done() blocked, but shouldn't have")
+ }
+ if e := child.Err(); e != Canceled {
+ t.Errorf("child.Err() == %v want %v", e, Canceled)
+ }
+
+ // parent should not be finished.
+ select {
+ case x := <-parent.Done():
+ t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
+ default:
+ }
+ if e := parent.Err(); e != nil {
+ t.Errorf("parent.Err() == %v want nil", e)
+ }
+ }
+}
+
+func testDeadline(c Context, wait time.Duration, t *testing.T) {
+ select {
+ case <-time.After(wait):
+ t.Fatalf("context should have timed out")
+ case <-c.Done():
+ }
+ if e := c.Err(); e != DeadlineExceeded {
+ t.Errorf("c.Err() == %v want %v", e, DeadlineExceeded)
+ }
+}
+
+func TestDeadline(t *testing.T) {
+ t.Parallel()
+ const timeUnit = 500 * time.Millisecond
+ c, _ := WithDeadline(Background(), time.Now().Add(1*timeUnit))
+ if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) {
+ t.Errorf("c.String() = %q want prefix %q", got, prefix)
+ }
+ testDeadline(c, 2*timeUnit, t)
+
+ c, _ = WithDeadline(Background(), time.Now().Add(1*timeUnit))
+ o := otherContext{c}
+ testDeadline(o, 2*timeUnit, t)
+
+ c, _ = WithDeadline(Background(), time.Now().Add(1*timeUnit))
+ o = otherContext{c}
+ c, _ = WithDeadline(o, time.Now().Add(3*timeUnit))
+ testDeadline(c, 2*timeUnit, t)
+}
+
+func TestTimeout(t *testing.T) {
+ t.Parallel()
+ const timeUnit = 500 * time.Millisecond
+ c, _ := WithTimeout(Background(), 1*timeUnit)
+ if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) {
+ t.Errorf("c.String() = %q want prefix %q", got, prefix)
+ }
+ testDeadline(c, 2*timeUnit, t)
+
+ c, _ = WithTimeout(Background(), 1*timeUnit)
+ o := otherContext{c}
+ testDeadline(o, 2*timeUnit, t)
+
+ c, _ = WithTimeout(Background(), 1*timeUnit)
+ o = otherContext{c}
+ c, _ = WithTimeout(o, 3*timeUnit)
+ testDeadline(c, 2*timeUnit, t)
+}
+
+func TestCanceledTimeout(t *testing.T) {
+ t.Parallel()
+ const timeUnit = 500 * time.Millisecond
+ c, _ := WithTimeout(Background(), 2*timeUnit)
+ o := otherContext{c}
+ c, cancel := WithTimeout(o, 4*timeUnit)
+ cancel()
+ time.Sleep(1 * timeUnit) // let cancelation propagate
+ select {
+ case <-c.Done():
+ default:
+ t.Errorf("<-c.Done() blocked, but shouldn't have")
+ }
+ if e := c.Err(); e != Canceled {
+ t.Errorf("c.Err() == %v want %v", e, Canceled)
+ }
+}
+
+type key1 int
+type key2 int
+
+var k1 = key1(1)
+var k2 = key2(1) // same int as k1, different type
+var k3 = key2(3) // same type as k2, different int
+
+func TestValues(t *testing.T) {
+ check := func(c Context, nm, v1, v2, v3 string) {
+ if v, ok := c.Value(k1).(string); ok == (len(v1) == 0) || v != v1 {
+ t.Errorf(`%s.Value(k1).(string) = %q, %t want %q, %t`, nm, v, ok, v1, len(v1) != 0)
+ }
+ if v, ok := c.Value(k2).(string); ok == (len(v2) == 0) || v != v2 {
+ t.Errorf(`%s.Value(k2).(string) = %q, %t want %q, %t`, nm, v, ok, v2, len(v2) != 0)
+ }
+ if v, ok := c.Value(k3).(string); ok == (len(v3) == 0) || v != v3 {
+ t.Errorf(`%s.Value(k3).(string) = %q, %t want %q, %t`, nm, v, ok, v3, len(v3) != 0)
+ }
+ }
+
+ c0 := Background()
+ check(c0, "c0", "", "", "")
+
+ c1 := WithValue(Background(), k1, "c1k1")
+ check(c1, "c1", "c1k1", "", "")
+
+ if got, want := fmt.Sprint(c1), `context.Background.WithValue(1, "c1k1")`; got != want {
+ t.Errorf("c.String() = %q want %q", got, want)
+ }
+
+ c2 := WithValue(c1, k2, "c2k2")
+ check(c2, "c2", "c1k1", "c2k2", "")
+
+ c3 := WithValue(c2, k3, "c3k3")
+ check(c3, "c2", "c1k1", "c2k2", "c3k3")
+
+ c4 := WithValue(c3, k1, nil)
+ check(c4, "c4", "", "c2k2", "c3k3")
+
+ o0 := otherContext{Background()}
+ check(o0, "o0", "", "", "")
+
+ o1 := otherContext{WithValue(Background(), k1, "c1k1")}
+ check(o1, "o1", "c1k1", "", "")
+
+ o2 := WithValue(o1, k2, "o2k2")
+ check(o2, "o2", "c1k1", "o2k2", "")
+
+ o3 := otherContext{c4}
+ check(o3, "o3", "", "c2k2", "c3k3")
+
+ o4 := WithValue(o3, k3, nil)
+ check(o4, "o4", "", "c2k2", "")
+}
+
+func TestAllocs(t *testing.T) {
+ bg := Background()
+ for _, test := range []struct {
+ desc string
+ f func()
+ limit float64
+ gccgoLimit float64
+ }{
+ {
+ desc: "Background()",
+ f: func() { Background() },
+ limit: 0,
+ gccgoLimit: 0,
+ },
+ {
+ desc: fmt.Sprintf("WithValue(bg, %v, nil)", k1),
+ f: func() {
+ c := WithValue(bg, k1, nil)
+ c.Value(k1)
+ },
+ limit: 3,
+ gccgoLimit: 3,
+ },
+ {
+ desc: "WithTimeout(bg, 15*time.Millisecond)",
+ f: func() {
+ c, _ := WithTimeout(bg, 15*time.Millisecond)
+ <-c.Done()
+ },
+ limit: 8,
+ gccgoLimit: 16,
+ },
+ {
+ desc: "WithCancel(bg)",
+ f: func() {
+ c, cancel := WithCancel(bg)
+ cancel()
+ <-c.Done()
+ },
+ limit: 5,
+ gccgoLimit: 8,
+ },
+ {
+ desc: "WithTimeout(bg, 100*time.Millisecond)",
+ f: func() {
+ c, cancel := WithTimeout(bg, 100*time.Millisecond)
+ cancel()
+ <-c.Done()
+ },
+ limit: 8,
+ gccgoLimit: 25,
+ },
+ } {
+ limit := test.limit
+ if runtime.Compiler == "gccgo" {
+ // gccgo does not yet do escape analysis.
+ // TODO(iant): Remove this when gccgo does do escape analysis.
+ limit = test.gccgoLimit
+ }
+ if n := testing.AllocsPerRun(100, test.f); n > limit {
+ t.Errorf("%s allocs = %f want %d", test.desc, n, int(limit))
+ }
+ }
+}
+
+func TestSimultaneousCancels(t *testing.T) {
+ root, cancel := WithCancel(Background())
+ m := map[Context]CancelFunc{root: cancel}
+ q := []Context{root}
+ // Create a tree of contexts.
+ for len(q) != 0 && len(m) < 100 {
+ parent := q[0]
+ q = q[1:]
+ for i := 0; i < 4; i++ {
+ ctx, cancel := WithCancel(parent)
+ m[ctx] = cancel
+ q = append(q, ctx)
+ }
+ }
+ // Start all the cancels in a random order.
+ var wg sync.WaitGroup
+ wg.Add(len(m))
+ for _, cancel := range m {
+ go func(cancel CancelFunc) {
+ cancel()
+ wg.Done()
+ }(cancel)
+ }
+ // Wait on all the contexts in a random order.
+ for ctx := range m {
+ select {
+ case <-ctx.Done():
+ case <-time.After(1 * time.Second):
+ buf := make([]byte, 10<<10)
+ n := runtime.Stack(buf, true)
+ t.Fatalf("timed out waiting for <-ctx.Done(); stacks:\n%s", buf[:n])
+ }
+ }
+ // Wait for all the cancel functions to return.
+ done := make(chan struct{})
+ go func() {
+ wg.Wait()
+ close(done)
+ }()
+ select {
+ case <-done:
+ case <-time.After(1 * time.Second):
+ buf := make([]byte, 10<<10)
+ n := runtime.Stack(buf, true)
+ t.Fatalf("timed out waiting for cancel functions; stacks:\n%s", buf[:n])
+ }
+}
+
+func TestInterlockedCancels(t *testing.T) {
+ parent, cancelParent := WithCancel(Background())
+ child, cancelChild := WithCancel(parent)
+ go func() {
+ parent.Done()
+ cancelChild()
+ }()
+ cancelParent()
+ select {
+ case <-child.Done():
+ case <-time.After(1 * time.Second):
+ buf := make([]byte, 10<<10)
+ n := runtime.Stack(buf, true)
+ t.Fatalf("timed out waiting for child.Done(); stacks:\n%s", buf[:n])
+ }
+}
+
+func TestLayersCancel(t *testing.T) {
+ testLayers(t, time.Now().UnixNano(), false)
+}
+
+func TestLayersTimeout(t *testing.T) {
+ testLayers(t, time.Now().UnixNano(), true)
+}
+
+func testLayers(t *testing.T, seed int64, testTimeout bool) {
+ rand.Seed(seed)
+ errorf := func(format string, a ...interface{}) {
+ t.Errorf(fmt.Sprintf("seed=%d: %s", seed, format), a...)
+ }
+ const (
+ timeout = 200 * time.Millisecond
+ minLayers = 30
+ )
+ type value int
+ var (
+ vals []*value
+ cancels []CancelFunc
+ numTimers int
+ ctx = Background()
+ )
+ for i := 0; i < minLayers || numTimers == 0 || len(cancels) == 0 || len(vals) == 0; i++ {
+ switch rand.Intn(3) {
+ case 0:
+ v := new(value)
+ ctx = WithValue(ctx, v, v)
+ vals = append(vals, v)
+ case 1:
+ var cancel CancelFunc
+ ctx, cancel = WithCancel(ctx)
+ cancels = append(cancels, cancel)
+ case 2:
+ var cancel CancelFunc
+ ctx, cancel = WithTimeout(ctx, timeout)
+ cancels = append(cancels, cancel)
+ numTimers++
+ }
+ }
+ checkValues := func(when string) {
+ for _, key := range vals {
+ if val := ctx.Value(key).(*value); key != val {
+ errorf("%s: ctx.Value(%p) = %p want %p", when, key, val, key)
+ }
+ }
+ }
+ select {
+ case <-ctx.Done():
+ errorf("ctx should not be canceled yet")
+ default:
+ }
+ if s, prefix := fmt.Sprint(ctx), "context.Background."; !strings.HasPrefix(s, prefix) {
+ t.Errorf("ctx.String() = %q want prefix %q", s, prefix)
+ }
+ t.Log(ctx)
+ checkValues("before cancel")
+ if testTimeout {
+ select {
+ case <-ctx.Done():
+ case <-time.After(timeout + 100*time.Millisecond):
+ errorf("ctx should have timed out")
+ }
+ checkValues("after timeout")
+ } else {
+ cancel := cancels[rand.Intn(len(cancels))]
+ cancel()
+ select {
+ case <-ctx.Done():
+ default:
+ errorf("ctx should be canceled")
+ }
+ checkValues("after cancel")
+ }
+}
+
+func TestCancelRemoves(t *testing.T) {
+ checkChildren := func(when string, ctx Context, want int) {
+ if got := len(ctx.(*cancelCtx).children); got != want {
+ t.Errorf("%s: context has %d children, want %d", when, got, want)
+ }
+ }
+
+ ctx, _ := WithCancel(Background())
+ checkChildren("after creation", ctx, 0)
+ _, cancel := WithCancel(ctx)
+ checkChildren("with WithCancel child ", ctx, 1)
+ cancel()
+ checkChildren("after cancelling WithCancel child", ctx, 0)
+
+ ctx, _ = WithCancel(Background())
+ checkChildren("after creation", ctx, 0)
+ _, cancel = WithTimeout(ctx, 60*time.Minute)
+ checkChildren("with WithTimeout child ", ctx, 1)
+ cancel()
+ checkChildren("after cancelling WithTimeout child", ctx, 0)
+}
diff --git a/vendor/golang.org/x/net/context/go17.go b/vendor/golang.org/x/net/context/go17.go
new file mode 100644
index 0000000..d20f52b
--- /dev/null
+++ b/vendor/golang.org/x/net/context/go17.go
@@ -0,0 +1,72 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.7
+
+package context
+
+import (
+ "context" // standard library's context, as of Go 1.7
+ "time"
+)
+
+var (
+ todo = context.TODO()
+ background = context.Background()
+)
+
+// Canceled is the error returned by Context.Err when the context is canceled.
+var Canceled = context.Canceled
+
+// DeadlineExceeded is the error returned by Context.Err when the context's
+// deadline passes.
+var DeadlineExceeded = context.DeadlineExceeded
+
+// WithCancel returns a copy of parent with a new Done channel. The returned
+// context's Done channel is closed when the returned cancel function is called
+// or when the parent context's Done channel is closed, whichever happens first.
+//
+// Canceling this context releases resources associated with it, so code should
+// call cancel as soon as the operations running in this Context complete.
+func WithCancel(parent Context) (ctx Context, cancel CancelFunc) {
+ ctx, f := context.WithCancel(parent)
+ return ctx, CancelFunc(f)
+}
+
+// WithDeadline returns a copy of the parent context with the deadline adjusted
+// to be no later than d. If the parent's deadline is already earlier than d,
+// WithDeadline(parent, d) is semantically equivalent to parent. The returned
+// context's Done channel is closed when the deadline expires, when the returned
+// cancel function is called, or when the parent context's Done channel is
+// closed, whichever happens first.
+//
+// Canceling this context releases resources associated with it, so code should
+// call cancel as soon as the operations running in this Context complete.
+func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) {
+ ctx, f := context.WithDeadline(parent, deadline)
+ return ctx, CancelFunc(f)
+}
+
+// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)).
+//
+// Canceling this context releases resources associated with it, so code should
+// call cancel as soon as the operations running in this Context complete:
+//
+// func slowOperationWithTimeout(ctx context.Context) (Result, error) {
+// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
+// defer cancel() // releases resources if slowOperation completes before timeout elapses
+// return slowOperation(ctx)
+// }
+func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) {
+ return WithDeadline(parent, time.Now().Add(timeout))
+}
+
+// WithValue returns a copy of parent in which the value associated with key is
+// val.
+//
+// Use context Values only for request-scoped data that transits processes and
+// APIs, not for passing optional parameters to functions.
+func WithValue(parent Context, key interface{}, val interface{}) Context {
+ return context.WithValue(parent, key, val)
+}
diff --git a/vendor/golang.org/x/net/context/go19.go b/vendor/golang.org/x/net/context/go19.go
new file mode 100644
index 0000000..d88bd1d
--- /dev/null
+++ b/vendor/golang.org/x/net/context/go19.go
@@ -0,0 +1,20 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.9
+
+package context
+
+import "context" // standard library's context, as of Go 1.7
+
+// A Context carries a deadline, a cancelation signal, and other values across
+// API boundaries.
+//
+// Context's methods may be called by multiple goroutines simultaneously.
+type Context = context.Context
+
+// A CancelFunc tells an operation to abandon its work.
+// A CancelFunc does not wait for the work to stop.
+// After the first call, subsequent calls to a CancelFunc do nothing.
+type CancelFunc = context.CancelFunc
diff --git a/vendor/golang.org/x/net/context/pre_go17.go b/vendor/golang.org/x/net/context/pre_go17.go
new file mode 100644
index 0000000..0f35592
--- /dev/null
+++ b/vendor/golang.org/x/net/context/pre_go17.go
@@ -0,0 +1,300 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !go1.7
+
+package context
+
+import (
+ "errors"
+ "fmt"
+ "sync"
+ "time"
+)
+
+// An emptyCtx is never canceled, has no values, and has no deadline. It is not
+// struct{}, since vars of this type must have distinct addresses.
+type emptyCtx int
+
+func (*emptyCtx) Deadline() (deadline time.Time, ok bool) {
+ return
+}
+
+func (*emptyCtx) Done() <-chan struct{} {
+ return nil
+}
+
+func (*emptyCtx) Err() error {
+ return nil
+}
+
+func (*emptyCtx) Value(key interface{}) interface{} {
+ return nil
+}
+
+func (e *emptyCtx) String() string {
+ switch e {
+ case background:
+ return "context.Background"
+ case todo:
+ return "context.TODO"
+ }
+ return "unknown empty Context"
+}
+
+var (
+ background = new(emptyCtx)
+ todo = new(emptyCtx)
+)
+
+// Canceled is the error returned by Context.Err when the context is canceled.
+var Canceled = errors.New("context canceled")
+
+// DeadlineExceeded is the error returned by Context.Err when the context's
+// deadline passes.
+var DeadlineExceeded = errors.New("context deadline exceeded")
+
+// WithCancel returns a copy of parent with a new Done channel. The returned
+// context's Done channel is closed when the returned cancel function is called
+// or when the parent context's Done channel is closed, whichever happens first.
+//
+// Canceling this context releases resources associated with it, so code should
+// call cancel as soon as the operations running in this Context complete.
+func WithCancel(parent Context) (ctx Context, cancel CancelFunc) {
+ c := newCancelCtx(parent)
+ propagateCancel(parent, c)
+ return c, func() { c.cancel(true, Canceled) }
+}
+
+// newCancelCtx returns an initialized cancelCtx.
+func newCancelCtx(parent Context) *cancelCtx {
+ return &cancelCtx{
+ Context: parent,
+ done: make(chan struct{}),
+ }
+}
+
+// propagateCancel arranges for child to be canceled when parent is.
+func propagateCancel(parent Context, child canceler) {
+ if parent.Done() == nil {
+ return // parent is never canceled
+ }
+ if p, ok := parentCancelCtx(parent); ok {
+ p.mu.Lock()
+ if p.err != nil {
+ // parent has already been canceled
+ child.cancel(false, p.err)
+ } else {
+ if p.children == nil {
+ p.children = make(map[canceler]bool)
+ }
+ p.children[child] = true
+ }
+ p.mu.Unlock()
+ } else {
+ go func() {
+ select {
+ case <-parent.Done():
+ child.cancel(false, parent.Err())
+ case <-child.Done():
+ }
+ }()
+ }
+}
+
+// parentCancelCtx follows a chain of parent references until it finds a
+// *cancelCtx. This function understands how each of the concrete types in this
+// package represents its parent.
+func parentCancelCtx(parent Context) (*cancelCtx, bool) {
+ for {
+ switch c := parent.(type) {
+ case *cancelCtx:
+ return c, true
+ case *timerCtx:
+ return c.cancelCtx, true
+ case *valueCtx:
+ parent = c.Context
+ default:
+ return nil, false
+ }
+ }
+}
+
+// removeChild removes a context from its parent.
+func removeChild(parent Context, child canceler) {
+ p, ok := parentCancelCtx(parent)
+ if !ok {
+ return
+ }
+ p.mu.Lock()
+ if p.children != nil {
+ delete(p.children, child)
+ }
+ p.mu.Unlock()
+}
+
+// A canceler is a context type that can be canceled directly. The
+// implementations are *cancelCtx and *timerCtx.
+type canceler interface {
+ cancel(removeFromParent bool, err error)
+ Done() <-chan struct{}
+}
+
+// A cancelCtx can be canceled. When canceled, it also cancels any children
+// that implement canceler.
+type cancelCtx struct {
+ Context
+
+ done chan struct{} // closed by the first cancel call.
+
+ mu sync.Mutex
+ children map[canceler]bool // set to nil by the first cancel call
+ err error // set to non-nil by the first cancel call
+}
+
+func (c *cancelCtx) Done() <-chan struct{} {
+ return c.done
+}
+
+func (c *cancelCtx) Err() error {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.err
+}
+
+func (c *cancelCtx) String() string {
+ return fmt.Sprintf("%v.WithCancel", c.Context)
+}
+
+// cancel closes c.done, cancels each of c's children, and, if
+// removeFromParent is true, removes c from its parent's children.
+func (c *cancelCtx) cancel(removeFromParent bool, err error) {
+ if err == nil {
+ panic("context: internal error: missing cancel error")
+ }
+ c.mu.Lock()
+ if c.err != nil {
+ c.mu.Unlock()
+ return // already canceled
+ }
+ c.err = err
+ close(c.done)
+ for child := range c.children {
+ // NOTE: acquiring the child's lock while holding parent's lock.
+ child.cancel(false, err)
+ }
+ c.children = nil
+ c.mu.Unlock()
+
+ if removeFromParent {
+ removeChild(c.Context, c)
+ }
+}
+
+// WithDeadline returns a copy of the parent context with the deadline adjusted
+// to be no later than d. If the parent's deadline is already earlier than d,
+// WithDeadline(parent, d) is semantically equivalent to parent. The returned
+// context's Done channel is closed when the deadline expires, when the returned
+// cancel function is called, or when the parent context's Done channel is
+// closed, whichever happens first.
+//
+// Canceling this context releases resources associated with it, so code should
+// call cancel as soon as the operations running in this Context complete.
+func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) {
+ if cur, ok := parent.Deadline(); ok && cur.Before(deadline) {
+ // The current deadline is already sooner than the new one.
+ return WithCancel(parent)
+ }
+ c := &timerCtx{
+ cancelCtx: newCancelCtx(parent),
+ deadline: deadline,
+ }
+ propagateCancel(parent, c)
+ d := deadline.Sub(time.Now())
+ if d <= 0 {
+ c.cancel(true, DeadlineExceeded) // deadline has already passed
+ return c, func() { c.cancel(true, Canceled) }
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.err == nil {
+ c.timer = time.AfterFunc(d, func() {
+ c.cancel(true, DeadlineExceeded)
+ })
+ }
+ return c, func() { c.cancel(true, Canceled) }
+}
+
+// A timerCtx carries a timer and a deadline. It embeds a cancelCtx to
+// implement Done and Err. It implements cancel by stopping its timer then
+// delegating to cancelCtx.cancel.
+type timerCtx struct {
+ *cancelCtx
+ timer *time.Timer // Under cancelCtx.mu.
+
+ deadline time.Time
+}
+
+func (c *timerCtx) Deadline() (deadline time.Time, ok bool) {
+ return c.deadline, true
+}
+
+func (c *timerCtx) String() string {
+ return fmt.Sprintf("%v.WithDeadline(%s [%s])", c.cancelCtx.Context, c.deadline, c.deadline.Sub(time.Now()))
+}
+
+func (c *timerCtx) cancel(removeFromParent bool, err error) {
+ c.cancelCtx.cancel(false, err)
+ if removeFromParent {
+ // Remove this timerCtx from its parent cancelCtx's children.
+ removeChild(c.cancelCtx.Context, c)
+ }
+ c.mu.Lock()
+ if c.timer != nil {
+ c.timer.Stop()
+ c.timer = nil
+ }
+ c.mu.Unlock()
+}
+
+// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)).
+//
+// Canceling this context releases resources associated with it, so code should
+// call cancel as soon as the operations running in this Context complete:
+//
+// func slowOperationWithTimeout(ctx context.Context) (Result, error) {
+// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
+// defer cancel() // releases resources if slowOperation completes before timeout elapses
+// return slowOperation(ctx)
+// }
+func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) {
+ return WithDeadline(parent, time.Now().Add(timeout))
+}
+
+// WithValue returns a copy of parent in which the value associated with key is
+// val.
+//
+// Use context Values only for request-scoped data that transits processes and
+// APIs, not for passing optional parameters to functions.
+func WithValue(parent Context, key interface{}, val interface{}) Context {
+ return &valueCtx{parent, key, val}
+}
+
+// A valueCtx carries a key-value pair. It implements Value for that key and
+// delegates all other calls to the embedded Context.
+type valueCtx struct {
+ Context
+ key, val interface{}
+}
+
+func (c *valueCtx) String() string {
+ return fmt.Sprintf("%v.WithValue(%#v, %#v)", c.Context, c.key, c.val)
+}
+
+func (c *valueCtx) Value(key interface{}) interface{} {
+ if c.key == key {
+ return c.val
+ }
+ return c.Context.Value(key)
+}
diff --git a/vendor/golang.org/x/net/context/pre_go19.go b/vendor/golang.org/x/net/context/pre_go19.go
new file mode 100644
index 0000000..b105f80
--- /dev/null
+++ b/vendor/golang.org/x/net/context/pre_go19.go
@@ -0,0 +1,109 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !go1.9
+
+package context
+
+import "time"
+
+// A Context carries a deadline, a cancelation signal, and other values across
+// API boundaries.
+//
+// Context's methods may be called by multiple goroutines simultaneously.
+type Context interface {
+ // Deadline returns the time when work done on behalf of this context
+ // should be canceled. Deadline returns ok==false when no deadline is
+ // set. Successive calls to Deadline return the same results.
+ Deadline() (deadline time.Time, ok bool)
+
+ // Done returns a channel that's closed when work done on behalf of this
+ // context should be canceled. Done may return nil if this context can
+ // never be canceled. Successive calls to Done return the same value.
+ //
+ // WithCancel arranges for Done to be closed when cancel is called;
+ // WithDeadline arranges for Done to be closed when the deadline
+ // expires; WithTimeout arranges for Done to be closed when the timeout
+ // elapses.
+ //
+ // Done is provided for use in select statements:
+ //
+ // // Stream generates values with DoSomething and sends them to out
+ // // until DoSomething returns an error or ctx.Done is closed.
+ // func Stream(ctx context.Context, out chan<- Value) error {
+ // for {
+ // v, err := DoSomething(ctx)
+ // if err != nil {
+ // return err
+ // }
+ // select {
+ // case <-ctx.Done():
+ // return ctx.Err()
+ // case out <- v:
+ // }
+ // }
+ // }
+ //
+ // See http://blog.golang.org/pipelines for more examples of how to use
+ // a Done channel for cancelation.
+ Done() <-chan struct{}
+
+ // Err returns a non-nil error value after Done is closed. Err returns
+ // Canceled if the context was canceled or DeadlineExceeded if the
+ // context's deadline passed. No other values for Err are defined.
+ // After Done is closed, successive calls to Err return the same value.
+ Err() error
+
+ // Value returns the value associated with this context for key, or nil
+ // if no value is associated with key. Successive calls to Value with
+ // the same key returns the same result.
+ //
+ // Use context values only for request-scoped data that transits
+ // processes and API boundaries, not for passing optional parameters to
+ // functions.
+ //
+ // A key identifies a specific value in a Context. Functions that wish
+ // to store values in Context typically allocate a key in a global
+ // variable then use that key as the argument to context.WithValue and
+ // Context.Value. A key can be any type that supports equality;
+ // packages should define keys as an unexported type to avoid
+ // collisions.
+ //
+ // Packages that define a Context key should provide type-safe accessors
+ // for the values stores using that key:
+ //
+ // // Package user defines a User type that's stored in Contexts.
+ // package user
+ //
+ // import "golang.org/x/net/context"
+ //
+ // // User is the type of value stored in the Contexts.
+ // type User struct {...}
+ //
+ // // key is an unexported type for keys defined in this package.
+ // // This prevents collisions with keys defined in other packages.
+ // type key int
+ //
+ // // userKey is the key for user.User values in Contexts. It is
+ // // unexported; clients use user.NewContext and user.FromContext
+ // // instead of using this key directly.
+ // var userKey key = 0
+ //
+ // // NewContext returns a new Context that carries value u.
+ // func NewContext(ctx context.Context, u *User) context.Context {
+ // return context.WithValue(ctx, userKey, u)
+ // }
+ //
+ // // FromContext returns the User value stored in ctx, if any.
+ // func FromContext(ctx context.Context) (*User, bool) {
+ // u, ok := ctx.Value(userKey).(*User)
+ // return u, ok
+ // }
+ Value(key interface{}) interface{}
+}
+
+// A CancelFunc tells an operation to abandon its work.
+// A CancelFunc does not wait for the work to stop.
+// After the first call, subsequent calls to a CancelFunc do nothing.
+type CancelFunc func()
diff --git a/vendor/golang.org/x/net/context/withtimeout_test.go b/vendor/golang.org/x/net/context/withtimeout_test.go
new file mode 100644
index 0000000..e6f5669
--- /dev/null
+++ b/vendor/golang.org/x/net/context/withtimeout_test.go
@@ -0,0 +1,31 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package context_test
+
+import (
+ "fmt"
+ "time"
+
+ "golang.org/x/net/context"
+)
+
+// This example passes a context with a timeout to tell a blocking function that
+// it should abandon its work after the timeout elapses.
+func ExampleWithTimeout() {
+ // Pass a context with a timeout to tell a blocking function that it
+ // should abandon its work after the timeout elapses.
+ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+ defer cancel()
+
+ select {
+ case <-time.After(1 * time.Second):
+ fmt.Println("overslept")
+ case <-ctx.Done():
+ fmt.Println(ctx.Err()) // prints "context deadline exceeded"
+ }
+
+ // Output:
+ // context deadline exceeded
+}
diff --git a/vendor/golang.org/x/net/html/atom/atom.go b/vendor/golang.org/x/net/html/atom/atom.go
new file mode 100644
index 0000000..cd0a8ac
--- /dev/null
+++ b/vendor/golang.org/x/net/html/atom/atom.go
@@ -0,0 +1,78 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package atom provides integer codes (also known as atoms) for a fixed set of
+// frequently occurring HTML strings: tag names and attribute keys such as "p"
+// and "id".
+//
+// Sharing an atom's name between all elements with the same tag can result in
+// fewer string allocations when tokenizing and parsing HTML. Integer
+// comparisons are also generally faster than string comparisons.
+//
+// The value of an atom's particular code is not guaranteed to stay the same
+// between versions of this package. Neither is any ordering guaranteed:
+// whether atom.H1 < atom.H2 may also change. The codes are not guaranteed to
+// be dense. The only guarantees are that e.g. looking up "div" will yield
+// atom.Div, calling atom.Div.String will return "div", and atom.Div != 0.
+package atom // import "golang.org/x/net/html/atom"
+
+// Atom is an integer code for a string. The zero value maps to "".
+type Atom uint32
+
+// String returns the atom's name.
+func (a Atom) String() string {
+ start := uint32(a >> 8)
+ n := uint32(a & 0xff)
+ if start+n > uint32(len(atomText)) {
+ return ""
+ }
+ return atomText[start : start+n]
+}
+
+func (a Atom) string() string {
+ return atomText[a>>8 : a>>8+a&0xff]
+}
+
+// fnv computes the FNV hash with an arbitrary starting value h.
+func fnv(h uint32, s []byte) uint32 {
+ for i := range s {
+ h ^= uint32(s[i])
+ h *= 16777619
+ }
+ return h
+}
+
+func match(s string, t []byte) bool {
+ for i, c := range t {
+ if s[i] != c {
+ return false
+ }
+ }
+ return true
+}
+
+// Lookup returns the atom whose name is s. It returns zero if there is no
+// such atom. The lookup is case sensitive.
+func Lookup(s []byte) Atom {
+ if len(s) == 0 || len(s) > maxAtomLen {
+ return 0
+ }
+ h := fnv(hash0, s)
+ if a := table[h&uint32(len(table)-1)]; int(a&0xff) == len(s) && match(a.string(), s) {
+ return a
+ }
+ if a := table[(h>>16)&uint32(len(table)-1)]; int(a&0xff) == len(s) && match(a.string(), s) {
+ return a
+ }
+ return 0
+}
+
+// String returns a string whose contents are equal to s. In that sense, it is
+// equivalent to string(s) but may be more efficient.
+func String(s []byte) string {
+ if a := Lookup(s); a != 0 {
+ return a.String()
+ }
+ return string(s)
+}
diff --git a/vendor/golang.org/x/net/html/atom/atom_test.go b/vendor/golang.org/x/net/html/atom/atom_test.go
new file mode 100644
index 0000000..6e33704
--- /dev/null
+++ b/vendor/golang.org/x/net/html/atom/atom_test.go
@@ -0,0 +1,109 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package atom
+
+import (
+ "sort"
+ "testing"
+)
+
+func TestKnown(t *testing.T) {
+ for _, s := range testAtomList {
+ if atom := Lookup([]byte(s)); atom.String() != s {
+ t.Errorf("Lookup(%q) = %#x (%q)", s, uint32(atom), atom.String())
+ }
+ }
+}
+
+func TestHits(t *testing.T) {
+ for _, a := range table {
+ if a == 0 {
+ continue
+ }
+ got := Lookup([]byte(a.String()))
+ if got != a {
+ t.Errorf("Lookup(%q) = %#x, want %#x", a.String(), uint32(got), uint32(a))
+ }
+ }
+}
+
+func TestMisses(t *testing.T) {
+ testCases := []string{
+ "",
+ "\x00",
+ "\xff",
+ "A",
+ "DIV",
+ "Div",
+ "dIV",
+ "aa",
+ "a\x00",
+ "ab",
+ "abb",
+ "abbr0",
+ "abbr ",
+ " abbr",
+ " a",
+ "acceptcharset",
+ "acceptCharset",
+ "accept_charset",
+ "h0",
+ "h1h2",
+ "h7",
+ "onClick",
+ "λ",
+ // The following string has the same hash (0xa1d7fab7) as "onmouseover".
+ "\x00\x00\x00\x00\x00\x50\x18\xae\x38\xd0\xb7",
+ }
+ for _, tc := range testCases {
+ got := Lookup([]byte(tc))
+ if got != 0 {
+ t.Errorf("Lookup(%q): got %d, want 0", tc, got)
+ }
+ }
+}
+
+func TestForeignObject(t *testing.T) {
+ const (
+ afo = Foreignobject
+ afO = ForeignObject
+ sfo = "foreignobject"
+ sfO = "foreignObject"
+ )
+ if got := Lookup([]byte(sfo)); got != afo {
+ t.Errorf("Lookup(%q): got %#v, want %#v", sfo, got, afo)
+ }
+ if got := Lookup([]byte(sfO)); got != afO {
+ t.Errorf("Lookup(%q): got %#v, want %#v", sfO, got, afO)
+ }
+ if got := afo.String(); got != sfo {
+ t.Errorf("Atom(%#v).String(): got %q, want %q", afo, got, sfo)
+ }
+ if got := afO.String(); got != sfO {
+ t.Errorf("Atom(%#v).String(): got %q, want %q", afO, got, sfO)
+ }
+}
+
+func BenchmarkLookup(b *testing.B) {
+ sortedTable := make([]string, 0, len(table))
+ for _, a := range table {
+ if a != 0 {
+ sortedTable = append(sortedTable, a.String())
+ }
+ }
+ sort.Strings(sortedTable)
+
+ x := make([][]byte, 1000)
+ for i := range x {
+ x[i] = []byte(sortedTable[i%len(sortedTable)])
+ }
+
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ for _, s := range x {
+ Lookup(s)
+ }
+ }
+}
diff --git a/vendor/golang.org/x/net/html/atom/table.go b/vendor/golang.org/x/net/html/atom/table.go
new file mode 100644
index 0000000..2a93886
--- /dev/null
+++ b/vendor/golang.org/x/net/html/atom/table.go
@@ -0,0 +1,783 @@
+// Code generated by go generate gen.go; DO NOT EDIT.
+
+//go:generate go run gen.go
+
+package atom
+
+const (
+ A Atom = 0x1
+ Abbr Atom = 0x4
+ Accept Atom = 0x1a06
+ AcceptCharset Atom = 0x1a0e
+ Accesskey Atom = 0x2c09
+ Acronym Atom = 0xaa07
+ Action Atom = 0x27206
+ Address Atom = 0x6f307
+ Align Atom = 0xb105
+ Allowfullscreen Atom = 0x2080f
+ Allowpaymentrequest Atom = 0xc113
+ Allowusermedia Atom = 0xdd0e
+ Alt Atom = 0xf303
+ Annotation Atom = 0x1c90a
+ AnnotationXml Atom = 0x1c90e
+ Applet Atom = 0x31906
+ Area Atom = 0x35604
+ Article Atom = 0x3fc07
+ As Atom = 0x3c02
+ Aside Atom = 0x10705
+ Async Atom = 0xff05
+ Audio Atom = 0x11505
+ Autocomplete Atom = 0x2780c
+ Autofocus Atom = 0x12109
+ Autoplay Atom = 0x13c08
+ B Atom = 0x101
+ Base Atom = 0x3b04
+ Basefont Atom = 0x3b08
+ Bdi Atom = 0xba03
+ Bdo Atom = 0x14b03
+ Bgsound Atom = 0x15e07
+ Big Atom = 0x17003
+ Blink Atom = 0x17305
+ Blockquote Atom = 0x1870a
+ Body Atom = 0x2804
+ Br Atom = 0x202
+ Button Atom = 0x19106
+ Canvas Atom = 0x10306
+ Caption Atom = 0x23107
+ Center Atom = 0x22006
+ Challenge Atom = 0x29b09
+ Charset Atom = 0x2107
+ Checked Atom = 0x47907
+ Cite Atom = 0x19c04
+ Class Atom = 0x56405
+ Code Atom = 0x5c504
+ Col Atom = 0x1ab03
+ Colgroup Atom = 0x1ab08
+ Color Atom = 0x1bf05
+ Cols Atom = 0x1c404
+ Colspan Atom = 0x1c407
+ Command Atom = 0x1d707
+ Content Atom = 0x58b07
+ Contenteditable Atom = 0x58b0f
+ Contextmenu Atom = 0x3800b
+ Controls Atom = 0x1de08
+ Coords Atom = 0x1ea06
+ Crossorigin Atom = 0x1fb0b
+ Data Atom = 0x4a504
+ Datalist Atom = 0x4a508
+ Datetime Atom = 0x2b808
+ Dd Atom = 0x2d702
+ Default Atom = 0x10a07
+ Defer Atom = 0x5c705
+ Del Atom = 0x45203
+ Desc Atom = 0x56104
+ Details Atom = 0x7207
+ Dfn Atom = 0x8703
+ Dialog Atom = 0xbb06
+ Dir Atom = 0x9303
+ Dirname Atom = 0x9307
+ Disabled Atom = 0x16408
+ Div Atom = 0x16b03
+ Dl Atom = 0x5e602
+ Download Atom = 0x46308
+ Draggable Atom = 0x17a09
+ Dropzone Atom = 0x40508
+ Dt Atom = 0x64b02
+ Em Atom = 0x6e02
+ Embed Atom = 0x6e05
+ Enctype Atom = 0x28d07
+ Face Atom = 0x21e04
+ Fieldset Atom = 0x22608
+ Figcaption Atom = 0x22e0a
+ Figure Atom = 0x24806
+ Font Atom = 0x3f04
+ Footer Atom = 0xf606
+ For Atom = 0x25403
+ ForeignObject Atom = 0x2540d
+ Foreignobject Atom = 0x2610d
+ Form Atom = 0x26e04
+ Formaction Atom = 0x26e0a
+ Formenctype Atom = 0x2890b
+ Formmethod Atom = 0x2a40a
+ Formnovalidate Atom = 0x2ae0e
+ Formtarget Atom = 0x2c00a
+ Frame Atom = 0x8b05
+ Frameset Atom = 0x8b08
+ H1 Atom = 0x15c02
+ H2 Atom = 0x2de02
+ H3 Atom = 0x30d02
+ H4 Atom = 0x34502
+ H5 Atom = 0x34f02
+ H6 Atom = 0x64d02
+ Head Atom = 0x33104
+ Header Atom = 0x33106
+ Headers Atom = 0x33107
+ Height Atom = 0x5206
+ Hgroup Atom = 0x2ca06
+ Hidden Atom = 0x2d506
+ High Atom = 0x2db04
+ Hr Atom = 0x15702
+ Href Atom = 0x2e004
+ Hreflang Atom = 0x2e008
+ Html Atom = 0x5604
+ HttpEquiv Atom = 0x2e80a
+ I Atom = 0x601
+ Icon Atom = 0x58a04
+ Id Atom = 0x10902
+ Iframe Atom = 0x2fc06
+ Image Atom = 0x30205
+ Img Atom = 0x30703
+ Input Atom = 0x44b05
+ Inputmode Atom = 0x44b09
+ Ins Atom = 0x20403
+ Integrity Atom = 0x23f09
+ Is Atom = 0x16502
+ Isindex Atom = 0x30f07
+ Ismap Atom = 0x31605
+ Itemid Atom = 0x38b06
+ Itemprop Atom = 0x19d08
+ Itemref Atom = 0x3cd07
+ Itemscope Atom = 0x67109
+ Itemtype Atom = 0x31f08
+ Kbd Atom = 0xb903
+ Keygen Atom = 0x3206
+ Keytype Atom = 0xd607
+ Kind Atom = 0x17704
+ Label Atom = 0x5905
+ Lang Atom = 0x2e404
+ Legend Atom = 0x18106
+ Li Atom = 0xb202
+ Link Atom = 0x17404
+ List Atom = 0x4a904
+ Listing Atom = 0x4a907
+ Loop Atom = 0x5d04
+ Low Atom = 0xc303
+ Main Atom = 0x1004
+ Malignmark Atom = 0xb00a
+ Manifest Atom = 0x6d708
+ Map Atom = 0x31803
+ Mark Atom = 0xb604
+ Marquee Atom = 0x32707
+ Math Atom = 0x32e04
+ Max Atom = 0x33d03
+ Maxlength Atom = 0x33d09
+ Media Atom = 0xe605
+ Mediagroup Atom = 0xe60a
+ Menu Atom = 0x38704
+ Menuitem Atom = 0x38708
+ Meta Atom = 0x4b804
+ Meter Atom = 0x9805
+ Method Atom = 0x2a806
+ Mglyph Atom = 0x30806
+ Mi Atom = 0x34702
+ Min Atom = 0x34703
+ Minlength Atom = 0x34709
+ Mn Atom = 0x2b102
+ Mo Atom = 0xa402
+ Ms Atom = 0x67402
+ Mtext Atom = 0x35105
+ Multiple Atom = 0x35f08
+ Muted Atom = 0x36705
+ Name Atom = 0x9604
+ Nav Atom = 0x1303
+ Nobr Atom = 0x3704
+ Noembed Atom = 0x6c07
+ Noframes Atom = 0x8908
+ Nomodule Atom = 0xa208
+ Nonce Atom = 0x1a605
+ Noscript Atom = 0x21608
+ Novalidate Atom = 0x2b20a
+ Object Atom = 0x26806
+ Ol Atom = 0x13702
+ Onabort Atom = 0x19507
+ Onafterprint Atom = 0x2360c
+ Onautocomplete Atom = 0x2760e
+ Onautocompleteerror Atom = 0x27613
+ Onauxclick Atom = 0x61f0a
+ Onbeforeprint Atom = 0x69e0d
+ Onbeforeunload Atom = 0x6e70e
+ Onblur Atom = 0x56d06
+ Oncancel Atom = 0x11908
+ Oncanplay Atom = 0x14d09
+ Oncanplaythrough Atom = 0x14d10
+ Onchange Atom = 0x41b08
+ Onclick Atom = 0x2f507
+ Onclose Atom = 0x36c07
+ Oncontextmenu Atom = 0x37e0d
+ Oncopy Atom = 0x39106
+ Oncuechange Atom = 0x3970b
+ Oncut Atom = 0x3a205
+ Ondblclick Atom = 0x3a70a
+ Ondrag Atom = 0x3b106
+ Ondragend Atom = 0x3b109
+ Ondragenter Atom = 0x3ba0b
+ Ondragexit Atom = 0x3c50a
+ Ondragleave Atom = 0x3df0b
+ Ondragover Atom = 0x3ea0a
+ Ondragstart Atom = 0x3f40b
+ Ondrop Atom = 0x40306
+ Ondurationchange Atom = 0x41310
+ Onemptied Atom = 0x40a09
+ Onended Atom = 0x42307
+ Onerror Atom = 0x42a07
+ Onfocus Atom = 0x43107
+ Onhashchange Atom = 0x43d0c
+ Oninput Atom = 0x44907
+ Oninvalid Atom = 0x45509
+ Onkeydown Atom = 0x45e09
+ Onkeypress Atom = 0x46b0a
+ Onkeyup Atom = 0x48007
+ Onlanguagechange Atom = 0x48d10
+ Onload Atom = 0x49d06
+ Onloadeddata Atom = 0x49d0c
+ Onloadedmetadata Atom = 0x4b010
+ Onloadend Atom = 0x4c609
+ Onloadstart Atom = 0x4cf0b
+ Onmessage Atom = 0x4da09
+ Onmessageerror Atom = 0x4da0e
+ Onmousedown Atom = 0x4e80b
+ Onmouseenter Atom = 0x4f30c
+ Onmouseleave Atom = 0x4ff0c
+ Onmousemove Atom = 0x50b0b
+ Onmouseout Atom = 0x5160a
+ Onmouseover Atom = 0x5230b
+ Onmouseup Atom = 0x52e09
+ Onmousewheel Atom = 0x53c0c
+ Onoffline Atom = 0x54809
+ Ononline Atom = 0x55108
+ Onpagehide Atom = 0x5590a
+ Onpageshow Atom = 0x5730a
+ Onpaste Atom = 0x57f07
+ Onpause Atom = 0x59a07
+ Onplay Atom = 0x5a406
+ Onplaying Atom = 0x5a409
+ Onpopstate Atom = 0x5ad0a
+ Onprogress Atom = 0x5b70a
+ Onratechange Atom = 0x5cc0c
+ Onrejectionhandled Atom = 0x5d812
+ Onreset Atom = 0x5ea07
+ Onresize Atom = 0x5f108
+ Onscroll Atom = 0x60008
+ Onsecuritypolicyviolation Atom = 0x60819
+ Onseeked Atom = 0x62908
+ Onseeking Atom = 0x63109
+ Onselect Atom = 0x63a08
+ Onshow Atom = 0x64406
+ Onsort Atom = 0x64f06
+ Onstalled Atom = 0x65909
+ Onstorage Atom = 0x66209
+ Onsubmit Atom = 0x66b08
+ Onsuspend Atom = 0x67b09
+ Ontimeupdate Atom = 0x400c
+ Ontoggle Atom = 0x68408
+ Onunhandledrejection Atom = 0x68c14
+ Onunload Atom = 0x6ab08
+ Onvolumechange Atom = 0x6b30e
+ Onwaiting Atom = 0x6c109
+ Onwheel Atom = 0x6ca07
+ Open Atom = 0x1a304
+ Optgroup Atom = 0x5f08
+ Optimum Atom = 0x6d107
+ Option Atom = 0x6e306
+ Output Atom = 0x51d06
+ P Atom = 0xc01
+ Param Atom = 0xc05
+ Pattern Atom = 0x6607
+ Picture Atom = 0x7b07
+ Ping Atom = 0xef04
+ Placeholder Atom = 0x1310b
+ Plaintext Atom = 0x1b209
+ Playsinline Atom = 0x1400b
+ Poster Atom = 0x2cf06
+ Pre Atom = 0x47003
+ Preload Atom = 0x48607
+ Progress Atom = 0x5b908
+ Prompt Atom = 0x53606
+ Public Atom = 0x58606
+ Q Atom = 0xcf01
+ Radiogroup Atom = 0x30a
+ Rb Atom = 0x3a02
+ Readonly Atom = 0x35708
+ Referrerpolicy Atom = 0x3d10e
+ Rel Atom = 0x48703
+ Required Atom = 0x24c08
+ Reversed Atom = 0x8008
+ Rows Atom = 0x9c04
+ Rowspan Atom = 0x9c07
+ Rp Atom = 0x23c02
+ Rt Atom = 0x19a02
+ Rtc Atom = 0x19a03
+ Ruby Atom = 0xfb04
+ S Atom = 0x2501
+ Samp Atom = 0x7804
+ Sandbox Atom = 0x12907
+ Scope Atom = 0x67505
+ Scoped Atom = 0x67506
+ Script Atom = 0x21806
+ Seamless Atom = 0x37108
+ Section Atom = 0x56807
+ Select Atom = 0x63c06
+ Selected Atom = 0x63c08
+ Shape Atom = 0x1e505
+ Size Atom = 0x5f504
+ Sizes Atom = 0x5f505
+ Slot Atom = 0x1ef04
+ Small Atom = 0x20605
+ Sortable Atom = 0x65108
+ Sorted Atom = 0x33706
+ Source Atom = 0x37806
+ Spacer Atom = 0x43706
+ Span Atom = 0x9f04
+ Spellcheck Atom = 0x4740a
+ Src Atom = 0x5c003
+ Srcdoc Atom = 0x5c006
+ Srclang Atom = 0x5f907
+ Srcset Atom = 0x6f906
+ Start Atom = 0x3fa05
+ Step Atom = 0x58304
+ Strike Atom = 0xd206
+ Strong Atom = 0x6dd06
+ Style Atom = 0x6ff05
+ Sub Atom = 0x66d03
+ Summary Atom = 0x70407
+ Sup Atom = 0x70b03
+ Svg Atom = 0x70e03
+ System Atom = 0x71106
+ Tabindex Atom = 0x4be08
+ Table Atom = 0x59505
+ Target Atom = 0x2c406
+ Tbody Atom = 0x2705
+ Td Atom = 0x9202
+ Template Atom = 0x71408
+ Textarea Atom = 0x35208
+ Tfoot Atom = 0xf505
+ Th Atom = 0x15602
+ Thead Atom = 0x33005
+ Time Atom = 0x4204
+ Title Atom = 0x11005
+ Tr Atom = 0xcc02
+ Track Atom = 0x1ba05
+ Translate Atom = 0x1f209
+ Tt Atom = 0x6802
+ Type Atom = 0xd904
+ Typemustmatch Atom = 0x2900d
+ U Atom = 0xb01
+ Ul Atom = 0xa702
+ Updateviacache Atom = 0x460e
+ Usemap Atom = 0x59e06
+ Value Atom = 0x1505
+ Var Atom = 0x16d03
+ Video Atom = 0x2f105
+ Wbr Atom = 0x57c03
+ Width Atom = 0x64905
+ Workertype Atom = 0x71c0a
+ Wrap Atom = 0x72604
+ Xmp Atom = 0x12f03
+)
+
+const hash0 = 0x81cdf10e
+
+const maxAtomLen = 25
+
+var table = [1 << 9]Atom{
+ 0x1: 0xe60a, // mediagroup
+ 0x2: 0x2e404, // lang
+ 0x4: 0x2c09, // accesskey
+ 0x5: 0x8b08, // frameset
+ 0x7: 0x63a08, // onselect
+ 0x8: 0x71106, // system
+ 0xa: 0x64905, // width
+ 0xc: 0x2890b, // formenctype
+ 0xd: 0x13702, // ol
+ 0xe: 0x3970b, // oncuechange
+ 0x10: 0x14b03, // bdo
+ 0x11: 0x11505, // audio
+ 0x12: 0x17a09, // draggable
+ 0x14: 0x2f105, // video
+ 0x15: 0x2b102, // mn
+ 0x16: 0x38704, // menu
+ 0x17: 0x2cf06, // poster
+ 0x19: 0xf606, // footer
+ 0x1a: 0x2a806, // method
+ 0x1b: 0x2b808, // datetime
+ 0x1c: 0x19507, // onabort
+ 0x1d: 0x460e, // updateviacache
+ 0x1e: 0xff05, // async
+ 0x1f: 0x49d06, // onload
+ 0x21: 0x11908, // oncancel
+ 0x22: 0x62908, // onseeked
+ 0x23: 0x30205, // image
+ 0x24: 0x5d812, // onrejectionhandled
+ 0x26: 0x17404, // link
+ 0x27: 0x51d06, // output
+ 0x28: 0x33104, // head
+ 0x29: 0x4ff0c, // onmouseleave
+ 0x2a: 0x57f07, // onpaste
+ 0x2b: 0x5a409, // onplaying
+ 0x2c: 0x1c407, // colspan
+ 0x2f: 0x1bf05, // color
+ 0x30: 0x5f504, // size
+ 0x31: 0x2e80a, // http-equiv
+ 0x33: 0x601, // i
+ 0x34: 0x5590a, // onpagehide
+ 0x35: 0x68c14, // onunhandledrejection
+ 0x37: 0x42a07, // onerror
+ 0x3a: 0x3b08, // basefont
+ 0x3f: 0x1303, // nav
+ 0x40: 0x17704, // kind
+ 0x41: 0x35708, // readonly
+ 0x42: 0x30806, // mglyph
+ 0x44: 0xb202, // li
+ 0x46: 0x2d506, // hidden
+ 0x47: 0x70e03, // svg
+ 0x48: 0x58304, // step
+ 0x49: 0x23f09, // integrity
+ 0x4a: 0x58606, // public
+ 0x4c: 0x1ab03, // col
+ 0x4d: 0x1870a, // blockquote
+ 0x4e: 0x34f02, // h5
+ 0x50: 0x5b908, // progress
+ 0x51: 0x5f505, // sizes
+ 0x52: 0x34502, // h4
+ 0x56: 0x33005, // thead
+ 0x57: 0xd607, // keytype
+ 0x58: 0x5b70a, // onprogress
+ 0x59: 0x44b09, // inputmode
+ 0x5a: 0x3b109, // ondragend
+ 0x5d: 0x3a205, // oncut
+ 0x5e: 0x43706, // spacer
+ 0x5f: 0x1ab08, // colgroup
+ 0x62: 0x16502, // is
+ 0x65: 0x3c02, // as
+ 0x66: 0x54809, // onoffline
+ 0x67: 0x33706, // sorted
+ 0x69: 0x48d10, // onlanguagechange
+ 0x6c: 0x43d0c, // onhashchange
+ 0x6d: 0x9604, // name
+ 0x6e: 0xf505, // tfoot
+ 0x6f: 0x56104, // desc
+ 0x70: 0x33d03, // max
+ 0x72: 0x1ea06, // coords
+ 0x73: 0x30d02, // h3
+ 0x74: 0x6e70e, // onbeforeunload
+ 0x75: 0x9c04, // rows
+ 0x76: 0x63c06, // select
+ 0x77: 0x9805, // meter
+ 0x78: 0x38b06, // itemid
+ 0x79: 0x53c0c, // onmousewheel
+ 0x7a: 0x5c006, // srcdoc
+ 0x7d: 0x1ba05, // track
+ 0x7f: 0x31f08, // itemtype
+ 0x82: 0xa402, // mo
+ 0x83: 0x41b08, // onchange
+ 0x84: 0x33107, // headers
+ 0x85: 0x5cc0c, // onratechange
+ 0x86: 0x60819, // onsecuritypolicyviolation
+ 0x88: 0x4a508, // datalist
+ 0x89: 0x4e80b, // onmousedown
+ 0x8a: 0x1ef04, // slot
+ 0x8b: 0x4b010, // onloadedmetadata
+ 0x8c: 0x1a06, // accept
+ 0x8d: 0x26806, // object
+ 0x91: 0x6b30e, // onvolumechange
+ 0x92: 0x2107, // charset
+ 0x93: 0x27613, // onautocompleteerror
+ 0x94: 0xc113, // allowpaymentrequest
+ 0x95: 0x2804, // body
+ 0x96: 0x10a07, // default
+ 0x97: 0x63c08, // selected
+ 0x98: 0x21e04, // face
+ 0x99: 0x1e505, // shape
+ 0x9b: 0x68408, // ontoggle
+ 0x9e: 0x64b02, // dt
+ 0x9f: 0xb604, // mark
+ 0xa1: 0xb01, // u
+ 0xa4: 0x6ab08, // onunload
+ 0xa5: 0x5d04, // loop
+ 0xa6: 0x16408, // disabled
+ 0xaa: 0x42307, // onended
+ 0xab: 0xb00a, // malignmark
+ 0xad: 0x67b09, // onsuspend
+ 0xae: 0x35105, // mtext
+ 0xaf: 0x64f06, // onsort
+ 0xb0: 0x19d08, // itemprop
+ 0xb3: 0x67109, // itemscope
+ 0xb4: 0x17305, // blink
+ 0xb6: 0x3b106, // ondrag
+ 0xb7: 0xa702, // ul
+ 0xb8: 0x26e04, // form
+ 0xb9: 0x12907, // sandbox
+ 0xba: 0x8b05, // frame
+ 0xbb: 0x1505, // value
+ 0xbc: 0x66209, // onstorage
+ 0xbf: 0xaa07, // acronym
+ 0xc0: 0x19a02, // rt
+ 0xc2: 0x202, // br
+ 0xc3: 0x22608, // fieldset
+ 0xc4: 0x2900d, // typemustmatch
+ 0xc5: 0xa208, // nomodule
+ 0xc6: 0x6c07, // noembed
+ 0xc7: 0x69e0d, // onbeforeprint
+ 0xc8: 0x19106, // button
+ 0xc9: 0x2f507, // onclick
+ 0xca: 0x70407, // summary
+ 0xcd: 0xfb04, // ruby
+ 0xce: 0x56405, // class
+ 0xcf: 0x3f40b, // ondragstart
+ 0xd0: 0x23107, // caption
+ 0xd4: 0xdd0e, // allowusermedia
+ 0xd5: 0x4cf0b, // onloadstart
+ 0xd9: 0x16b03, // div
+ 0xda: 0x4a904, // list
+ 0xdb: 0x32e04, // math
+ 0xdc: 0x44b05, // input
+ 0xdf: 0x3ea0a, // ondragover
+ 0xe0: 0x2de02, // h2
+ 0xe2: 0x1b209, // plaintext
+ 0xe4: 0x4f30c, // onmouseenter
+ 0xe7: 0x47907, // checked
+ 0xe8: 0x47003, // pre
+ 0xea: 0x35f08, // multiple
+ 0xeb: 0xba03, // bdi
+ 0xec: 0x33d09, // maxlength
+ 0xed: 0xcf01, // q
+ 0xee: 0x61f0a, // onauxclick
+ 0xf0: 0x57c03, // wbr
+ 0xf2: 0x3b04, // base
+ 0xf3: 0x6e306, // option
+ 0xf5: 0x41310, // ondurationchange
+ 0xf7: 0x8908, // noframes
+ 0xf9: 0x40508, // dropzone
+ 0xfb: 0x67505, // scope
+ 0xfc: 0x8008, // reversed
+ 0xfd: 0x3ba0b, // ondragenter
+ 0xfe: 0x3fa05, // start
+ 0xff: 0x12f03, // xmp
+ 0x100: 0x5f907, // srclang
+ 0x101: 0x30703, // img
+ 0x104: 0x101, // b
+ 0x105: 0x25403, // for
+ 0x106: 0x10705, // aside
+ 0x107: 0x44907, // oninput
+ 0x108: 0x35604, // area
+ 0x109: 0x2a40a, // formmethod
+ 0x10a: 0x72604, // wrap
+ 0x10c: 0x23c02, // rp
+ 0x10d: 0x46b0a, // onkeypress
+ 0x10e: 0x6802, // tt
+ 0x110: 0x34702, // mi
+ 0x111: 0x36705, // muted
+ 0x112: 0xf303, // alt
+ 0x113: 0x5c504, // code
+ 0x114: 0x6e02, // em
+ 0x115: 0x3c50a, // ondragexit
+ 0x117: 0x9f04, // span
+ 0x119: 0x6d708, // manifest
+ 0x11a: 0x38708, // menuitem
+ 0x11b: 0x58b07, // content
+ 0x11d: 0x6c109, // onwaiting
+ 0x11f: 0x4c609, // onloadend
+ 0x121: 0x37e0d, // oncontextmenu
+ 0x123: 0x56d06, // onblur
+ 0x124: 0x3fc07, // article
+ 0x125: 0x9303, // dir
+ 0x126: 0xef04, // ping
+ 0x127: 0x24c08, // required
+ 0x128: 0x45509, // oninvalid
+ 0x129: 0xb105, // align
+ 0x12b: 0x58a04, // icon
+ 0x12c: 0x64d02, // h6
+ 0x12d: 0x1c404, // cols
+ 0x12e: 0x22e0a, // figcaption
+ 0x12f: 0x45e09, // onkeydown
+ 0x130: 0x66b08, // onsubmit
+ 0x131: 0x14d09, // oncanplay
+ 0x132: 0x70b03, // sup
+ 0x133: 0xc01, // p
+ 0x135: 0x40a09, // onemptied
+ 0x136: 0x39106, // oncopy
+ 0x137: 0x19c04, // cite
+ 0x138: 0x3a70a, // ondblclick
+ 0x13a: 0x50b0b, // onmousemove
+ 0x13c: 0x66d03, // sub
+ 0x13d: 0x48703, // rel
+ 0x13e: 0x5f08, // optgroup
+ 0x142: 0x9c07, // rowspan
+ 0x143: 0x37806, // source
+ 0x144: 0x21608, // noscript
+ 0x145: 0x1a304, // open
+ 0x146: 0x20403, // ins
+ 0x147: 0x2540d, // foreignObject
+ 0x148: 0x5ad0a, // onpopstate
+ 0x14a: 0x28d07, // enctype
+ 0x14b: 0x2760e, // onautocomplete
+ 0x14c: 0x35208, // textarea
+ 0x14e: 0x2780c, // autocomplete
+ 0x14f: 0x15702, // hr
+ 0x150: 0x1de08, // controls
+ 0x151: 0x10902, // id
+ 0x153: 0x2360c, // onafterprint
+ 0x155: 0x2610d, // foreignobject
+ 0x156: 0x32707, // marquee
+ 0x157: 0x59a07, // onpause
+ 0x158: 0x5e602, // dl
+ 0x159: 0x5206, // height
+ 0x15a: 0x34703, // min
+ 0x15b: 0x9307, // dirname
+ 0x15c: 0x1f209, // translate
+ 0x15d: 0x5604, // html
+ 0x15e: 0x34709, // minlength
+ 0x15f: 0x48607, // preload
+ 0x160: 0x71408, // template
+ 0x161: 0x3df0b, // ondragleave
+ 0x162: 0x3a02, // rb
+ 0x164: 0x5c003, // src
+ 0x165: 0x6dd06, // strong
+ 0x167: 0x7804, // samp
+ 0x168: 0x6f307, // address
+ 0x169: 0x55108, // ononline
+ 0x16b: 0x1310b, // placeholder
+ 0x16c: 0x2c406, // target
+ 0x16d: 0x20605, // small
+ 0x16e: 0x6ca07, // onwheel
+ 0x16f: 0x1c90a, // annotation
+ 0x170: 0x4740a, // spellcheck
+ 0x171: 0x7207, // details
+ 0x172: 0x10306, // canvas
+ 0x173: 0x12109, // autofocus
+ 0x174: 0xc05, // param
+ 0x176: 0x46308, // download
+ 0x177: 0x45203, // del
+ 0x178: 0x36c07, // onclose
+ 0x179: 0xb903, // kbd
+ 0x17a: 0x31906, // applet
+ 0x17b: 0x2e004, // href
+ 0x17c: 0x5f108, // onresize
+ 0x17e: 0x49d0c, // onloadeddata
+ 0x180: 0xcc02, // tr
+ 0x181: 0x2c00a, // formtarget
+ 0x182: 0x11005, // title
+ 0x183: 0x6ff05, // style
+ 0x184: 0xd206, // strike
+ 0x185: 0x59e06, // usemap
+ 0x186: 0x2fc06, // iframe
+ 0x187: 0x1004, // main
+ 0x189: 0x7b07, // picture
+ 0x18c: 0x31605, // ismap
+ 0x18e: 0x4a504, // data
+ 0x18f: 0x5905, // label
+ 0x191: 0x3d10e, // referrerpolicy
+ 0x192: 0x15602, // th
+ 0x194: 0x53606, // prompt
+ 0x195: 0x56807, // section
+ 0x197: 0x6d107, // optimum
+ 0x198: 0x2db04, // high
+ 0x199: 0x15c02, // h1
+ 0x19a: 0x65909, // onstalled
+ 0x19b: 0x16d03, // var
+ 0x19c: 0x4204, // time
+ 0x19e: 0x67402, // ms
+ 0x19f: 0x33106, // header
+ 0x1a0: 0x4da09, // onmessage
+ 0x1a1: 0x1a605, // nonce
+ 0x1a2: 0x26e0a, // formaction
+ 0x1a3: 0x22006, // center
+ 0x1a4: 0x3704, // nobr
+ 0x1a5: 0x59505, // table
+ 0x1a6: 0x4a907, // listing
+ 0x1a7: 0x18106, // legend
+ 0x1a9: 0x29b09, // challenge
+ 0x1aa: 0x24806, // figure
+ 0x1ab: 0xe605, // media
+ 0x1ae: 0xd904, // type
+ 0x1af: 0x3f04, // font
+ 0x1b0: 0x4da0e, // onmessageerror
+ 0x1b1: 0x37108, // seamless
+ 0x1b2: 0x8703, // dfn
+ 0x1b3: 0x5c705, // defer
+ 0x1b4: 0xc303, // low
+ 0x1b5: 0x19a03, // rtc
+ 0x1b6: 0x5230b, // onmouseover
+ 0x1b7: 0x2b20a, // novalidate
+ 0x1b8: 0x71c0a, // workertype
+ 0x1ba: 0x3cd07, // itemref
+ 0x1bd: 0x1, // a
+ 0x1be: 0x31803, // map
+ 0x1bf: 0x400c, // ontimeupdate
+ 0x1c0: 0x15e07, // bgsound
+ 0x1c1: 0x3206, // keygen
+ 0x1c2: 0x2705, // tbody
+ 0x1c5: 0x64406, // onshow
+ 0x1c7: 0x2501, // s
+ 0x1c8: 0x6607, // pattern
+ 0x1cc: 0x14d10, // oncanplaythrough
+ 0x1ce: 0x2d702, // dd
+ 0x1cf: 0x6f906, // srcset
+ 0x1d0: 0x17003, // big
+ 0x1d2: 0x65108, // sortable
+ 0x1d3: 0x48007, // onkeyup
+ 0x1d5: 0x5a406, // onplay
+ 0x1d7: 0x4b804, // meta
+ 0x1d8: 0x40306, // ondrop
+ 0x1da: 0x60008, // onscroll
+ 0x1db: 0x1fb0b, // crossorigin
+ 0x1dc: 0x5730a, // onpageshow
+ 0x1dd: 0x4, // abbr
+ 0x1de: 0x9202, // td
+ 0x1df: 0x58b0f, // contenteditable
+ 0x1e0: 0x27206, // action
+ 0x1e1: 0x1400b, // playsinline
+ 0x1e2: 0x43107, // onfocus
+ 0x1e3: 0x2e008, // hreflang
+ 0x1e5: 0x5160a, // onmouseout
+ 0x1e6: 0x5ea07, // onreset
+ 0x1e7: 0x13c08, // autoplay
+ 0x1e8: 0x63109, // onseeking
+ 0x1ea: 0x67506, // scoped
+ 0x1ec: 0x30a, // radiogroup
+ 0x1ee: 0x3800b, // contextmenu
+ 0x1ef: 0x52e09, // onmouseup
+ 0x1f1: 0x2ca06, // hgroup
+ 0x1f2: 0x2080f, // allowfullscreen
+ 0x1f3: 0x4be08, // tabindex
+ 0x1f6: 0x30f07, // isindex
+ 0x1f7: 0x1a0e, // accept-charset
+ 0x1f8: 0x2ae0e, // formnovalidate
+ 0x1fb: 0x1c90e, // annotation-xml
+ 0x1fc: 0x6e05, // embed
+ 0x1fd: 0x21806, // script
+ 0x1fe: 0xbb06, // dialog
+ 0x1ff: 0x1d707, // command
+}
+
+const atomText = "abbradiogrouparamainavalueaccept-charsetbodyaccesskeygenobrb" +
+ "asefontimeupdateviacacheightmlabelooptgroupatternoembedetail" +
+ "sampictureversedfnoframesetdirnameterowspanomoduleacronymali" +
+ "gnmarkbdialogallowpaymentrequestrikeytypeallowusermediagroup" +
+ "ingaltfooterubyasyncanvasidefaultitleaudioncancelautofocusan" +
+ "dboxmplaceholderautoplaysinlinebdoncanplaythrough1bgsoundisa" +
+ "bledivarbigblinkindraggablegendblockquotebuttonabortcitempro" +
+ "penoncecolgrouplaintextrackcolorcolspannotation-xmlcommandco" +
+ "ntrolshapecoordslotranslatecrossoriginsmallowfullscreenoscri" +
+ "ptfacenterfieldsetfigcaptionafterprintegrityfigurequiredfore" +
+ "ignObjectforeignobjectformactionautocompleteerrorformenctype" +
+ "mustmatchallengeformmethodformnovalidatetimeformtargethgroup" +
+ "osterhiddenhigh2hreflanghttp-equivideonclickiframeimageimgly" +
+ "ph3isindexismappletitemtypemarqueematheadersortedmaxlength4m" +
+ "inlength5mtextareadonlymultiplemutedoncloseamlessourceoncont" +
+ "extmenuitemidoncopyoncuechangeoncutondblclickondragendondrag" +
+ "enterondragexitemreferrerpolicyondragleaveondragoverondragst" +
+ "articleondropzonemptiedondurationchangeonendedonerroronfocus" +
+ "paceronhashchangeoninputmodeloninvalidonkeydownloadonkeypres" +
+ "spellcheckedonkeyupreloadonlanguagechangeonloadeddatalisting" +
+ "onloadedmetadatabindexonloadendonloadstartonmessageerroronmo" +
+ "usedownonmouseenteronmouseleaveonmousemoveonmouseoutputonmou" +
+ "seoveronmouseupromptonmousewheelonofflineononlineonpagehides" +
+ "classectionbluronpageshowbronpastepublicontenteditableonpaus" +
+ "emaponplayingonpopstateonprogressrcdocodeferonratechangeonre" +
+ "jectionhandledonresetonresizesrclangonscrollonsecuritypolicy" +
+ "violationauxclickonseekedonseekingonselectedonshowidth6onsor" +
+ "tableonstalledonstorageonsubmitemscopedonsuspendontoggleonun" +
+ "handledrejectionbeforeprintonunloadonvolumechangeonwaitingon" +
+ "wheeloptimumanifestrongoptionbeforeunloaddressrcsetstylesumm" +
+ "arysupsvgsystemplateworkertypewrap"
diff --git a/vendor/golang.org/x/net/html/atom/table_test.go b/vendor/golang.org/x/net/html/atom/table_test.go
new file mode 100644
index 0000000..8a30762
--- /dev/null
+++ b/vendor/golang.org/x/net/html/atom/table_test.go
@@ -0,0 +1,376 @@
+// Code generated by go generate gen.go; DO NOT EDIT.
+
+//go:generate go run gen.go -test
+
+package atom
+
+var testAtomList = []string{
+ "a",
+ "abbr",
+ "accept",
+ "accept-charset",
+ "accesskey",
+ "acronym",
+ "action",
+ "address",
+ "align",
+ "allowfullscreen",
+ "allowpaymentrequest",
+ "allowusermedia",
+ "alt",
+ "annotation",
+ "annotation-xml",
+ "applet",
+ "area",
+ "article",
+ "as",
+ "aside",
+ "async",
+ "audio",
+ "autocomplete",
+ "autofocus",
+ "autoplay",
+ "b",
+ "base",
+ "basefont",
+ "bdi",
+ "bdo",
+ "bgsound",
+ "big",
+ "blink",
+ "blockquote",
+ "body",
+ "br",
+ "button",
+ "canvas",
+ "caption",
+ "center",
+ "challenge",
+ "charset",
+ "checked",
+ "cite",
+ "class",
+ "code",
+ "col",
+ "colgroup",
+ "color",
+ "cols",
+ "colspan",
+ "command",
+ "content",
+ "contenteditable",
+ "contextmenu",
+ "controls",
+ "coords",
+ "crossorigin",
+ "data",
+ "datalist",
+ "datetime",
+ "dd",
+ "default",
+ "defer",
+ "del",
+ "desc",
+ "details",
+ "dfn",
+ "dialog",
+ "dir",
+ "dirname",
+ "disabled",
+ "div",
+ "dl",
+ "download",
+ "draggable",
+ "dropzone",
+ "dt",
+ "em",
+ "embed",
+ "enctype",
+ "face",
+ "fieldset",
+ "figcaption",
+ "figure",
+ "font",
+ "footer",
+ "for",
+ "foreignObject",
+ "foreignobject",
+ "form",
+ "formaction",
+ "formenctype",
+ "formmethod",
+ "formnovalidate",
+ "formtarget",
+ "frame",
+ "frameset",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "head",
+ "header",
+ "headers",
+ "height",
+ "hgroup",
+ "hidden",
+ "high",
+ "hr",
+ "href",
+ "hreflang",
+ "html",
+ "http-equiv",
+ "i",
+ "icon",
+ "id",
+ "iframe",
+ "image",
+ "img",
+ "input",
+ "inputmode",
+ "ins",
+ "integrity",
+ "is",
+ "isindex",
+ "ismap",
+ "itemid",
+ "itemprop",
+ "itemref",
+ "itemscope",
+ "itemtype",
+ "kbd",
+ "keygen",
+ "keytype",
+ "kind",
+ "label",
+ "lang",
+ "legend",
+ "li",
+ "link",
+ "list",
+ "listing",
+ "loop",
+ "low",
+ "main",
+ "malignmark",
+ "manifest",
+ "map",
+ "mark",
+ "marquee",
+ "math",
+ "max",
+ "maxlength",
+ "media",
+ "mediagroup",
+ "menu",
+ "menuitem",
+ "meta",
+ "meter",
+ "method",
+ "mglyph",
+ "mi",
+ "min",
+ "minlength",
+ "mn",
+ "mo",
+ "ms",
+ "mtext",
+ "multiple",
+ "muted",
+ "name",
+ "nav",
+ "nobr",
+ "noembed",
+ "noframes",
+ "nomodule",
+ "nonce",
+ "noscript",
+ "novalidate",
+ "object",
+ "ol",
+ "onabort",
+ "onafterprint",
+ "onautocomplete",
+ "onautocompleteerror",
+ "onauxclick",
+ "onbeforeprint",
+ "onbeforeunload",
+ "onblur",
+ "oncancel",
+ "oncanplay",
+ "oncanplaythrough",
+ "onchange",
+ "onclick",
+ "onclose",
+ "oncontextmenu",
+ "oncopy",
+ "oncuechange",
+ "oncut",
+ "ondblclick",
+ "ondrag",
+ "ondragend",
+ "ondragenter",
+ "ondragexit",
+ "ondragleave",
+ "ondragover",
+ "ondragstart",
+ "ondrop",
+ "ondurationchange",
+ "onemptied",
+ "onended",
+ "onerror",
+ "onfocus",
+ "onhashchange",
+ "oninput",
+ "oninvalid",
+ "onkeydown",
+ "onkeypress",
+ "onkeyup",
+ "onlanguagechange",
+ "onload",
+ "onloadeddata",
+ "onloadedmetadata",
+ "onloadend",
+ "onloadstart",
+ "onmessage",
+ "onmessageerror",
+ "onmousedown",
+ "onmouseenter",
+ "onmouseleave",
+ "onmousemove",
+ "onmouseout",
+ "onmouseover",
+ "onmouseup",
+ "onmousewheel",
+ "onoffline",
+ "ononline",
+ "onpagehide",
+ "onpageshow",
+ "onpaste",
+ "onpause",
+ "onplay",
+ "onplaying",
+ "onpopstate",
+ "onprogress",
+ "onratechange",
+ "onrejectionhandled",
+ "onreset",
+ "onresize",
+ "onscroll",
+ "onsecuritypolicyviolation",
+ "onseeked",
+ "onseeking",
+ "onselect",
+ "onshow",
+ "onsort",
+ "onstalled",
+ "onstorage",
+ "onsubmit",
+ "onsuspend",
+ "ontimeupdate",
+ "ontoggle",
+ "onunhandledrejection",
+ "onunload",
+ "onvolumechange",
+ "onwaiting",
+ "onwheel",
+ "open",
+ "optgroup",
+ "optimum",
+ "option",
+ "output",
+ "p",
+ "param",
+ "pattern",
+ "picture",
+ "ping",
+ "placeholder",
+ "plaintext",
+ "playsinline",
+ "poster",
+ "pre",
+ "preload",
+ "progress",
+ "prompt",
+ "public",
+ "q",
+ "radiogroup",
+ "rb",
+ "readonly",
+ "referrerpolicy",
+ "rel",
+ "required",
+ "reversed",
+ "rows",
+ "rowspan",
+ "rp",
+ "rt",
+ "rtc",
+ "ruby",
+ "s",
+ "samp",
+ "sandbox",
+ "scope",
+ "scoped",
+ "script",
+ "seamless",
+ "section",
+ "select",
+ "selected",
+ "shape",
+ "size",
+ "sizes",
+ "slot",
+ "small",
+ "sortable",
+ "sorted",
+ "source",
+ "spacer",
+ "span",
+ "spellcheck",
+ "src",
+ "srcdoc",
+ "srclang",
+ "srcset",
+ "start",
+ "step",
+ "strike",
+ "strong",
+ "style",
+ "sub",
+ "summary",
+ "sup",
+ "svg",
+ "system",
+ "tabindex",
+ "table",
+ "target",
+ "tbody",
+ "td",
+ "template",
+ "textarea",
+ "tfoot",
+ "th",
+ "thead",
+ "time",
+ "title",
+ "tr",
+ "track",
+ "translate",
+ "tt",
+ "type",
+ "typemustmatch",
+ "u",
+ "ul",
+ "updateviacache",
+ "usemap",
+ "value",
+ "var",
+ "video",
+ "wbr",
+ "width",
+ "workertype",
+ "wrap",
+ "xmp",
+}
diff --git a/vendor/golang.org/x/net/html/charset/charset.go b/vendor/golang.org/x/net/html/charset/charset.go
new file mode 100644
index 0000000..13bed15
--- /dev/null
+++ b/vendor/golang.org/x/net/html/charset/charset.go
@@ -0,0 +1,257 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package charset provides common text encodings for HTML documents.
+//
+// The mapping from encoding labels to encodings is defined at
+// https://encoding.spec.whatwg.org/.
+package charset // import "golang.org/x/net/html/charset"
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "mime"
+ "strings"
+ "unicode/utf8"
+
+ "golang.org/x/net/html"
+ "golang.org/x/text/encoding"
+ "golang.org/x/text/encoding/charmap"
+ "golang.org/x/text/encoding/htmlindex"
+ "golang.org/x/text/transform"
+)
+
+// Lookup returns the encoding with the specified label, and its canonical
+// name. It returns nil and the empty string if label is not one of the
+// standard encodings for HTML. Matching is case-insensitive and ignores
+// leading and trailing whitespace. Encoders will use HTML escape sequences for
+// runes that are not supported by the character set.
+func Lookup(label string) (e encoding.Encoding, name string) {
+ e, err := htmlindex.Get(label)
+ if err != nil {
+ return nil, ""
+ }
+ name, _ = htmlindex.Name(e)
+ return &htmlEncoding{e}, name
+}
+
+type htmlEncoding struct{ encoding.Encoding }
+
+func (h *htmlEncoding) NewEncoder() *encoding.Encoder {
+ // HTML requires a non-terminating legacy encoder. We use HTML escapes to
+ // substitute unsupported code points.
+ return encoding.HTMLEscapeUnsupported(h.Encoding.NewEncoder())
+}
+
+// DetermineEncoding determines the encoding of an HTML document by examining
+// up to the first 1024 bytes of content and the declared Content-Type.
+//
+// See http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#determining-the-character-encoding
+func DetermineEncoding(content []byte, contentType string) (e encoding.Encoding, name string, certain bool) {
+ if len(content) > 1024 {
+ content = content[:1024]
+ }
+
+ for _, b := range boms {
+ if bytes.HasPrefix(content, b.bom) {
+ e, name = Lookup(b.enc)
+ return e, name, true
+ }
+ }
+
+ if _, params, err := mime.ParseMediaType(contentType); err == nil {
+ if cs, ok := params["charset"]; ok {
+ if e, name = Lookup(cs); e != nil {
+ return e, name, true
+ }
+ }
+ }
+
+ if len(content) > 0 {
+ e, name = prescan(content)
+ if e != nil {
+ return e, name, false
+ }
+ }
+
+ // Try to detect UTF-8.
+ // First eliminate any partial rune at the end.
+ for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- {
+ b := content[i]
+ if b < 0x80 {
+ break
+ }
+ if utf8.RuneStart(b) {
+ content = content[:i]
+ break
+ }
+ }
+ hasHighBit := false
+ for _, c := range content {
+ if c >= 0x80 {
+ hasHighBit = true
+ break
+ }
+ }
+ if hasHighBit && utf8.Valid(content) {
+ return encoding.Nop, "utf-8", false
+ }
+
+ // TODO: change default depending on user's locale?
+ return charmap.Windows1252, "windows-1252", false
+}
+
+// NewReader returns an io.Reader that converts the content of r to UTF-8.
+// It calls DetermineEncoding to find out what r's encoding is.
+func NewReader(r io.Reader, contentType string) (io.Reader, error) {
+ preview := make([]byte, 1024)
+ n, err := io.ReadFull(r, preview)
+ switch {
+ case err == io.ErrUnexpectedEOF:
+ preview = preview[:n]
+ r = bytes.NewReader(preview)
+ case err != nil:
+ return nil, err
+ default:
+ r = io.MultiReader(bytes.NewReader(preview), r)
+ }
+
+ if e, _, _ := DetermineEncoding(preview, contentType); e != encoding.Nop {
+ r = transform.NewReader(r, e.NewDecoder())
+ }
+ return r, nil
+}
+
+// NewReaderLabel returns a reader that converts from the specified charset to
+// UTF-8. It uses Lookup to find the encoding that corresponds to label, and
+// returns an error if Lookup returns nil. It is suitable for use as
+// encoding/xml.Decoder's CharsetReader function.
+func NewReaderLabel(label string, input io.Reader) (io.Reader, error) {
+ e, _ := Lookup(label)
+ if e == nil {
+ return nil, fmt.Errorf("unsupported charset: %q", label)
+ }
+ return transform.NewReader(input, e.NewDecoder()), nil
+}
+
+func prescan(content []byte) (e encoding.Encoding, name string) {
+ z := html.NewTokenizer(bytes.NewReader(content))
+ for {
+ switch z.Next() {
+ case html.ErrorToken:
+ return nil, ""
+
+ case html.StartTagToken, html.SelfClosingTagToken:
+ tagName, hasAttr := z.TagName()
+ if !bytes.Equal(tagName, []byte("meta")) {
+ continue
+ }
+ attrList := make(map[string]bool)
+ gotPragma := false
+
+ const (
+ dontKnow = iota
+ doNeedPragma
+ doNotNeedPragma
+ )
+ needPragma := dontKnow
+
+ name = ""
+ e = nil
+ for hasAttr {
+ var key, val []byte
+ key, val, hasAttr = z.TagAttr()
+ ks := string(key)
+ if attrList[ks] {
+ continue
+ }
+ attrList[ks] = true
+ for i, c := range val {
+ if 'A' <= c && c <= 'Z' {
+ val[i] = c + 0x20
+ }
+ }
+
+ switch ks {
+ case "http-equiv":
+ if bytes.Equal(val, []byte("content-type")) {
+ gotPragma = true
+ }
+
+ case "content":
+ if e == nil {
+ name = fromMetaElement(string(val))
+ if name != "" {
+ e, name = Lookup(name)
+ if e != nil {
+ needPragma = doNeedPragma
+ }
+ }
+ }
+
+ case "charset":
+ e, name = Lookup(string(val))
+ needPragma = doNotNeedPragma
+ }
+ }
+
+ if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma {
+ continue
+ }
+
+ if strings.HasPrefix(name, "utf-16") {
+ name = "utf-8"
+ e = encoding.Nop
+ }
+
+ if e != nil {
+ return e, name
+ }
+ }
+ }
+}
+
+func fromMetaElement(s string) string {
+ for s != "" {
+ csLoc := strings.Index(s, "charset")
+ if csLoc == -1 {
+ return ""
+ }
+ s = s[csLoc+len("charset"):]
+ s = strings.TrimLeft(s, " \t\n\f\r")
+ if !strings.HasPrefix(s, "=") {
+ continue
+ }
+ s = s[1:]
+ s = strings.TrimLeft(s, " \t\n\f\r")
+ if s == "" {
+ return ""
+ }
+ if q := s[0]; q == '"' || q == '\'' {
+ s = s[1:]
+ closeQuote := strings.IndexRune(s, rune(q))
+ if closeQuote == -1 {
+ return ""
+ }
+ return s[:closeQuote]
+ }
+
+ end := strings.IndexAny(s, "; \t\n\f\r")
+ if end == -1 {
+ end = len(s)
+ }
+ return s[:end]
+ }
+ return ""
+}
+
+var boms = []struct {
+ bom []byte
+ enc string
+}{
+ {[]byte{0xfe, 0xff}, "utf-16be"},
+ {[]byte{0xff, 0xfe}, "utf-16le"},
+ {[]byte{0xef, 0xbb, 0xbf}, "utf-8"},
+}
diff --git a/vendor/golang.org/x/net/html/charset/charset_test.go b/vendor/golang.org/x/net/html/charset/charset_test.go
new file mode 100644
index 0000000..e4e7d86
--- /dev/null
+++ b/vendor/golang.org/x/net/html/charset/charset_test.go
@@ -0,0 +1,237 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package charset
+
+import (
+ "bytes"
+ "encoding/xml"
+ "io/ioutil"
+ "runtime"
+ "strings"
+ "testing"
+
+ "golang.org/x/text/transform"
+)
+
+func transformString(t transform.Transformer, s string) (string, error) {
+ r := transform.NewReader(strings.NewReader(s), t)
+ b, err := ioutil.ReadAll(r)
+ return string(b), err
+}
+
+type testCase struct {
+ utf8, other, otherEncoding string
+}
+
+// testCases for encoding and decoding.
+var testCases = []testCase{
+ {"Résumé", "Résumé", "utf8"},
+ {"Résumé", "R\xe9sum\xe9", "latin1"},
+ {"ã“ã‚Œã¯æ¼¢å—ã§ã™ã€‚", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"},
+ {"ã“ã‚Œã¯æ¼¢å—ã§ã™ã€‚", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"},
+ {"Hello, world", "Hello, world", "ASCII"},
+ {"Gdańsk", "Gda\xf1sk", "ISO-8859-2"},
+ {"Ââ ÄŒÄ ÄÄ‘ ÅŠÅ‹ Õõ Å Å¡ Žž Ã…Ã¥ Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"},
+ {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"},
+ {"latviešu", "latvie\xf0u", "ISO-8859-13"},
+ {"Seònaid", "Se\xf2naid", "ISO-8859-14"},
+ {"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"},
+ {"românește", "rom\xe2ne\xbate", "ISO-8859-16"},
+ {"nutraĵo", "nutra\xbco", "ISO-8859-3"},
+ {"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"},
+ {"руÑÑкий", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"},
+ {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"},
+ {"KaÄŸan", "Ka\xf0an", "ISO-8859-9"},
+ {"Résumé", "R\x8esum\x8e", "macintosh"},
+ {"Gdańsk", "Gda\xf1sk", "windows-1250"},
+ {"руÑÑкий", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"},
+ {"Résumé", "R\xe9sum\xe9", "windows-1252"},
+ {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"},
+ {"KaÄŸan", "Ka\xf0an", "windows-1254"},
+ {"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"},
+ {"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"},
+ {"latviešu", "latvie\xf0u", "windows-1257"},
+ {"Việt", "Vi\xea\xf2t", "windows-1258"},
+ {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"},
+ {"руÑÑкий", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"},
+ {"українÑька", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"},
+ {"Hello 常用國å—標準å—體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"},
+ {"Hello 常用國å—標準å—體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"},
+ {"Hello 常用國å—標準å—體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"},
+ {"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"},
+ {"㧯", "\x82\x31\x89\x38", "gb18030"},
+ {"ã“ã‚Œã¯æ¼¢å—ã§ã™ã€‚", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"},
+ {"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"},
+ {"ｲｳｴｵｶ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"},
+ {"ã“ã‚Œã¯æ¼¢å—ã§ã™ã€‚", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"},
+ {"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"},
+ {"다ìŒê³¼ ê°™ì€ ì¡°ê±´ì„ ë”°ë¼ì•¼ 합니다: ì €ìž‘ìží‘œì‹œ", "\xb4\xd9\xc0\xbd\xb0\xfa \xb0\xb0\xc0\xba \xc1\xb6\xb0\xc7\xc0\xbb \xb5\xfb\xb6\xf3\xbe\xdf \xc7Õ´Ï´\xd9: \xc0\xfa\xc0\xdb\xc0\xdaÇ¥\xbd\xc3", "EUC-KR"},
+}
+
+func TestDecode(t *testing.T) {
+ testCases := append(testCases, []testCase{
+ // Replace multi-byte maximum subpart of ill-formed subsequence with
+ // single replacement character (WhatWG requirement).
+ {"Rés\ufffdumé", "Rés\xe1\x80umé", "utf8"},
+ }...)
+ for _, tc := range testCases {
+ e, _ := Lookup(tc.otherEncoding)
+ if e == nil {
+ t.Errorf("%s: not found", tc.otherEncoding)
+ continue
+ }
+ s, err := transformString(e.NewDecoder(), tc.other)
+ if err != nil {
+ t.Errorf("%s: decode %q: %v", tc.otherEncoding, tc.other, err)
+ continue
+ }
+ if s != tc.utf8 {
+ t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.utf8)
+ }
+ }
+}
+
+func TestEncode(t *testing.T) {
+ testCases := append(testCases, []testCase{
+ // Use Go-style replacement.
+ {"Rés\xe1\x80umé", "Rés\ufffd\ufffdumé", "utf8"},
+ // U+0144 LATIN SMALL LETTER N WITH ACUTE not supported by encoding.
+ {"GdaÅ„sk", "Gdańsk", "ISO-8859-11"},
+ {"\ufffd", "�", "ISO-8859-11"},
+ {"a\xe1\x80b", "a��b", "ISO-8859-11"},
+ }...)
+ for _, tc := range testCases {
+ e, _ := Lookup(tc.otherEncoding)
+ if e == nil {
+ t.Errorf("%s: not found", tc.otherEncoding)
+ continue
+ }
+ s, err := transformString(e.NewEncoder(), tc.utf8)
+ if err != nil {
+ t.Errorf("%s: encode %q: %s", tc.otherEncoding, tc.utf8, err)
+ continue
+ }
+ if s != tc.other {
+ t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.other)
+ }
+ }
+}
+
+var sniffTestCases = []struct {
+ filename, declared, want string
+}{
+ {"HTTP-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
+ {"UTF-16LE-BOM.html", "", "utf-16le"},
+ {"UTF-16BE-BOM.html", "", "utf-16be"},
+ {"meta-content-attribute.html", "text/html", "iso-8859-15"},
+ {"meta-charset-attribute.html", "text/html", "iso-8859-15"},
+ {"No-encoding-declaration.html", "text/html", "utf-8"},
+ {"HTTP-vs-UTF-8-BOM.html", "text/html; charset=iso-8859-15", "utf-8"},
+ {"HTTP-vs-meta-content.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
+ {"HTTP-vs-meta-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
+ {"UTF-8-BOM-vs-meta-content.html", "text/html", "utf-8"},
+ {"UTF-8-BOM-vs-meta-charset.html", "text/html", "utf-8"},
+}
+
+func TestSniff(t *testing.T) {
+ switch runtime.GOOS {
+ case "nacl": // platforms that don't permit direct file system access
+ t.Skipf("not supported on %q", runtime.GOOS)
+ }
+
+ for _, tc := range sniffTestCases {
+ content, err := ioutil.ReadFile("testdata/" + tc.filename)
+ if err != nil {
+ t.Errorf("%s: error reading file: %v", tc.filename, err)
+ continue
+ }
+
+ _, name, _ := DetermineEncoding(content, tc.declared)
+ if name != tc.want {
+ t.Errorf("%s: got %q, want %q", tc.filename, name, tc.want)
+ continue
+ }
+ }
+}
+
+func TestReader(t *testing.T) {
+ switch runtime.GOOS {
+ case "nacl": // platforms that don't permit direct file system access
+ t.Skipf("not supported on %q", runtime.GOOS)
+ }
+
+ for _, tc := range sniffTestCases {
+ content, err := ioutil.ReadFile("testdata/" + tc.filename)
+ if err != nil {
+ t.Errorf("%s: error reading file: %v", tc.filename, err)
+ continue
+ }
+
+ r, err := NewReader(bytes.NewReader(content), tc.declared)
+ if err != nil {
+ t.Errorf("%s: error creating reader: %v", tc.filename, err)
+ continue
+ }
+
+ got, err := ioutil.ReadAll(r)
+ if err != nil {
+ t.Errorf("%s: error reading from charset.NewReader: %v", tc.filename, err)
+ continue
+ }
+
+ e, _ := Lookup(tc.want)
+ want, err := ioutil.ReadAll(transform.NewReader(bytes.NewReader(content), e.NewDecoder()))
+ if err != nil {
+ t.Errorf("%s: error decoding with hard-coded charset name: %v", tc.filename, err)
+ continue
+ }
+
+ if !bytes.Equal(got, want) {
+ t.Errorf("%s: got %q, want %q", tc.filename, got, want)
+ continue
+ }
+ }
+}
+
+var metaTestCases = []struct {
+ meta, want string
+}{
+ {"", ""},
+ {"text/html", ""},
+ {"text/html; charset utf-8", ""},
+ {"text/html; charset=latin-2", "latin-2"},
+ {"text/html; charset; charset = utf-8", "utf-8"},
+ {`charset="big5"`, "big5"},
+ {"charset='shift_jis'", "shift_jis"},
+}
+
+func TestFromMeta(t *testing.T) {
+ for _, tc := range metaTestCases {
+ got := fromMetaElement(tc.meta)
+ if got != tc.want {
+ t.Errorf("%q: got %q, want %q", tc.meta, got, tc.want)
+ }
+ }
+}
+
+func TestXML(t *testing.T) {
+ const s = "
HTTP charset
+ + + + + +The character encoding of a page can be set using the HTTP header charset declaration.
+The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.
The only character encoding declaration for this HTML file is in the HTTP header, which sets the encoding to ISO 8859-15.
+the-input-byte-stream-001
Result summary & related tests
Detailed results for this test
Link to spec
HTTP vs UTF-8 BOM
+ + + + + +A character encoding set in the HTTP header has lower precedence than the UTF-8 signature.
+The HTTP header attempts to set the character encoding to ISO 8859-15. The page starts with a UTF-8 signature.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.
If the test is unsuccessful, the characters  should appear at the top of the page. These represent the bytes that make up the UTF-8 signature when encountered in the ISO 8859-15 encoding.
+the-input-byte-stream-034
Result summary & related tests
Detailed results for this test
Link to spec
HTTP vs meta charset
+ + + + + +The HTTP header has a higher precedence than an encoding declaration in a meta charset attribute.
+The HTTP header attempts to set the character encoding to ISO 8859-15. The page contains an encoding declaration in a meta charset attribute that attempts to set the character encoding to ISO 8859-1.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.
the-input-byte-stream-018
Result summary & related tests
Detailed results for this test
Link to spec
HTTP vs meta content
+ + + + + +The HTTP header has a higher precedence than an encoding declaration in a meta content attribute.
+The HTTP header attempts to set the character encoding to ISO 8859-15. The page contains an encoding declaration in a meta content attribute that attempts to set the character encoding to ISO 8859-1.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.
the-input-byte-stream-016
Result summary & related tests
Detailed results for this test
Link to spec
No encoding declaration
+ + + + + +A page with no encoding information in HTTP, BOM, XML declaration or meta element will be treated as UTF-8.
+The test on this page contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.
the-input-byte-stream-015
Result summary & related tests
Detailed results for this test
Link to spec
UTF-8 BOM vs meta charset
+ + + + + +A page with a UTF-8 BOM will be recognized as UTF-8 even if the meta charset attribute declares a different encoding.
+The page contains an encoding declaration in a meta charset attribute that attempts to set the character encoding to ISO 8859-15, but the file starts with a UTF-8 signature.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.
the-input-byte-stream-038
Result summary & related tests
Detailed results for this test
Link to spec
UTF-8 BOM vs meta content
+ + + + + +A page with a UTF-8 BOM will be recognized as UTF-8 even if the meta content attribute declares a different encoding.
+The page contains an encoding declaration in a meta content attribute that attempts to set the character encoding to ISO 8859-15, but the file starts with a UTF-8 signature.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.
the-input-byte-stream-037
Result summary & related tests
Detailed results for this test
Link to spec
meta charset attribute
+ + + + + +The character encoding of the page can be set by a meta element with charset attribute.
+The only character encoding declaration for this HTML file is in the charset attribute of the meta element, which declares the encoding to be ISO 8859-15.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.
the-input-byte-stream-009
Result summary & related tests
Detailed results for this test
Link to spec
meta content attribute
+ + + + + +The character encoding of the page can be set by a meta element with http-equiv and content attributes.
+The only character encoding declaration for this HTML file is in the content attribute of the meta element, which declares the encoding to be ISO 8859-15.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.
the-input-byte-stream-007
Result summary & related tests
Detailed results for this test
Link to spec
Links:
` + doc, err := html.Parse(strings.NewReader(s)) + if err != nil { + log.Fatal(err) + } + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "a" { + for _, a := range n.Attr { + if a.Key == "href" { + fmt.Println(a.Val) + break + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(doc) + // Output: + // foo + // /bar/baz +} diff --git a/vendor/golang.org/x/net/html/foreign.go b/vendor/golang.org/x/net/html/foreign.go new file mode 100644 index 0000000..01477a9 --- /dev/null +++ b/vendor/golang.org/x/net/html/foreign.go @@ -0,0 +1,226 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "strings" +) + +func adjustAttributeNames(aa []Attribute, nameMap map[string]string) { + for i := range aa { + if newName, ok := nameMap[aa[i].Key]; ok { + aa[i].Key = newName + } + } +} + +func adjustForeignAttributes(aa []Attribute) { + for i, a := range aa { + if a.Key == "" || a.Key[0] != 'x' { + continue + } + switch a.Key { + case "xlink:actuate", "xlink:arcrole", "xlink:href", "xlink:role", "xlink:show", + "xlink:title", "xlink:type", "xml:base", "xml:lang", "xml:space", "xmlns:xlink": + j := strings.Index(a.Key, ":") + aa[i].Namespace = a.Key[:j] + aa[i].Key = a.Key[j+1:] + } + } +} + +func htmlIntegrationPoint(n *Node) bool { + if n.Type != ElementNode { + return false + } + switch n.Namespace { + case "math": + if n.Data == "annotation-xml" { + for _, a := range n.Attr { + if a.Key == "encoding" { + val := strings.ToLower(a.Val) + if val == "text/html" || val == "application/xhtml+xml" { + return true + } + } + } + } + case "svg": + switch n.Data { + case "desc", "foreignObject", "title": + return true + } + } + return false +} + +func mathMLTextIntegrationPoint(n *Node) bool { + if n.Namespace != "math" { + return false + } + switch n.Data { + case "mi", "mo", "mn", "ms", "mtext": + return true + } + return false +} + +// Section 12.2.6.5. +var breakout = map[string]bool{ + "b": true, + "big": true, + "blockquote": true, + "body": true, + "br": true, + "center": true, + "code": true, + "dd": true, + "div": true, + "dl": true, + "dt": true, + "em": true, + "embed": true, + "h1": true, + "h2": true, + "h3": true, + "h4": true, + "h5": true, + "h6": true, + "head": true, + "hr": true, + "i": true, + "img": true, + "li": true, + "listing": true, + "menu": true, + "meta": true, + "nobr": true, + "ol": true, + "p": true, + "pre": true, + "ruby": true, + "s": true, + "small": true, + "span": true, + "strong": true, + "strike": true, + "sub": true, + "sup": true, + "table": true, + "tt": true, + "u": true, + "ul": true, + "var": true, +} + +// Section 12.2.6.5. +var svgTagNameAdjustments = map[string]string{ + "altglyph": "altGlyph", + "altglyphdef": "altGlyphDef", + "altglyphitem": "altGlyphItem", + "animatecolor": "animateColor", + "animatemotion": "animateMotion", + "animatetransform": "animateTransform", + "clippath": "clipPath", + "feblend": "feBlend", + "fecolormatrix": "feColorMatrix", + "fecomponenttransfer": "feComponentTransfer", + "fecomposite": "feComposite", + "feconvolvematrix": "feConvolveMatrix", + "fediffuselighting": "feDiffuseLighting", + "fedisplacementmap": "feDisplacementMap", + "fedistantlight": "feDistantLight", + "feflood": "feFlood", + "fefunca": "feFuncA", + "fefuncb": "feFuncB", + "fefuncg": "feFuncG", + "fefuncr": "feFuncR", + "fegaussianblur": "feGaussianBlur", + "feimage": "feImage", + "femerge": "feMerge", + "femergenode": "feMergeNode", + "femorphology": "feMorphology", + "feoffset": "feOffset", + "fepointlight": "fePointLight", + "fespecularlighting": "feSpecularLighting", + "fespotlight": "feSpotLight", + "fetile": "feTile", + "feturbulence": "feTurbulence", + "foreignobject": "foreignObject", + "glyphref": "glyphRef", + "lineargradient": "linearGradient", + "radialgradient": "radialGradient", + "textpath": "textPath", +} + +// Section 12.2.6.1 +var mathMLAttributeAdjustments = map[string]string{ + "definitionurl": "definitionURL", +} + +var svgAttributeAdjustments = map[string]string{ + "attributename": "attributeName", + "attributetype": "attributeType", + "basefrequency": "baseFrequency", + "baseprofile": "baseProfile", + "calcmode": "calcMode", + "clippathunits": "clipPathUnits", + "contentscripttype": "contentScriptType", + "contentstyletype": "contentStyleType", + "diffuseconstant": "diffuseConstant", + "edgemode": "edgeMode", + "externalresourcesrequired": "externalResourcesRequired", + "filterres": "filterRes", + "filterunits": "filterUnits", + "glyphref": "glyphRef", + "gradienttransform": "gradientTransform", + "gradientunits": "gradientUnits", + "kernelmatrix": "kernelMatrix", + "kernelunitlength": "kernelUnitLength", + "keypoints": "keyPoints", + "keysplines": "keySplines", + "keytimes": "keyTimes", + "lengthadjust": "lengthAdjust", + "limitingconeangle": "limitingConeAngle", + "markerheight": "markerHeight", + "markerunits": "markerUnits", + "markerwidth": "markerWidth", + "maskcontentunits": "maskContentUnits", + "maskunits": "maskUnits", + "numoctaves": "numOctaves", + "pathlength": "pathLength", + "patterncontentunits": "patternContentUnits", + "patterntransform": "patternTransform", + "patternunits": "patternUnits", + "pointsatx": "pointsAtX", + "pointsaty": "pointsAtY", + "pointsatz": "pointsAtZ", + "preservealpha": "preserveAlpha", + "preserveaspectratio": "preserveAspectRatio", + "primitiveunits": "primitiveUnits", + "refx": "refX", + "refy": "refY", + "repeatcount": "repeatCount", + "repeatdur": "repeatDur", + "requiredextensions": "requiredExtensions", + "requiredfeatures": "requiredFeatures", + "specularconstant": "specularConstant", + "specularexponent": "specularExponent", + "spreadmethod": "spreadMethod", + "startoffset": "startOffset", + "stddeviation": "stdDeviation", + "stitchtiles": "stitchTiles", + "surfacescale": "surfaceScale", + "systemlanguage": "systemLanguage", + "tablevalues": "tableValues", + "targetx": "targetX", + "targety": "targetY", + "textlength": "textLength", + "viewbox": "viewBox", + "viewtarget": "viewTarget", + "xchannelselector": "xChannelSelector", + "ychannelselector": "yChannelSelector", + "zoomandpan": "zoomAndPan", +} diff --git a/vendor/golang.org/x/net/html/node.go b/vendor/golang.org/x/net/html/node.go new file mode 100644 index 0000000..2c1cade --- /dev/null +++ b/vendor/golang.org/x/net/html/node.go @@ -0,0 +1,220 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "golang.org/x/net/html/atom" +) + +// A NodeType is the type of a Node. +type NodeType uint32 + +const ( + ErrorNode NodeType = iota + TextNode + DocumentNode + ElementNode + CommentNode + DoctypeNode + scopeMarkerNode +) + +// Section 12.2.4.3 says "The markers are inserted when entering applet, +// object, marquee, template, td, th, and caption elements, and are used +// to prevent formatting from "leaking" into applet, object, marquee, +// template, td, th, and caption elements". +var scopeMarker = Node{Type: scopeMarkerNode} + +// A Node consists of a NodeType and some Data (tag name for element nodes, +// content for text) and are part of a tree of Nodes. Element nodes may also +// have a Namespace and contain a slice of Attributes. Data is unescaped, so +// that it looks like "a block.
+ if d != "" && d[0] == '\r' {
+ d = d[1:]
+ }
+ if d != "" && d[0] == '\n' {
+ d = d[1:]
+ }
+ }
+ }
+ d = strings.Replace(d, "\x00", "", -1)
+ if d == "" {
+ return true
+ }
+ p.reconstructActiveFormattingElements()
+ p.addText(d)
+ if p.framesetOK && strings.TrimLeft(d, whitespace) != "" {
+ // There were non-whitespace characters inserted.
+ p.framesetOK = false
+ }
+ case StartTagToken:
+ switch p.tok.DataAtom {
+ case a.Html:
+ if p.oe.contains(a.Template) {
+ return true
+ }
+ copyAttributes(p.oe[0], p.tok)
+ case a.Base, a.Basefont, a.Bgsound, a.Command, a.Link, a.Meta, a.Noframes, a.Script, a.Style, a.Template, a.Title:
+ return inHeadIM(p)
+ case a.Body:
+ if p.oe.contains(a.Template) {
+ return true
+ }
+ if len(p.oe) >= 2 {
+ body := p.oe[1]
+ if body.Type == ElementNode && body.DataAtom == a.Body {
+ p.framesetOK = false
+ copyAttributes(body, p.tok)
+ }
+ }
+ case a.Frameset:
+ if !p.framesetOK || len(p.oe) < 2 || p.oe[1].DataAtom != a.Body {
+ // Ignore the token.
+ return true
+ }
+ body := p.oe[1]
+ if body.Parent != nil {
+ body.Parent.RemoveChild(body)
+ }
+ p.oe = p.oe[:1]
+ p.addElement()
+ p.im = inFramesetIM
+ return true
+ case a.Address, a.Article, a.Aside, a.Blockquote, a.Center, a.Details, a.Dir, a.Div, a.Dl, a.Fieldset, a.Figcaption, a.Figure, a.Footer, a.Header, a.Hgroup, a.Menu, a.Nav, a.Ol, a.P, a.Section, a.Summary, a.Ul:
+ p.popUntil(buttonScope, a.P)
+ p.addElement()
+ case a.H1, a.H2, a.H3, a.H4, a.H5, a.H6:
+ p.popUntil(buttonScope, a.P)
+ switch n := p.top(); n.DataAtom {
+ case a.H1, a.H2, a.H3, a.H4, a.H5, a.H6:
+ p.oe.pop()
+ }
+ p.addElement()
+ case a.Pre, a.Listing:
+ p.popUntil(buttonScope, a.P)
+ p.addElement()
+ // The newline, if any, will be dealt with by the TextToken case.
+ p.framesetOK = false
+ case a.Form:
+ if p.form != nil && !p.oe.contains(a.Template) {
+ // Ignore the token
+ return true
+ }
+ p.popUntil(buttonScope, a.P)
+ p.addElement()
+ if !p.oe.contains(a.Template) {
+ p.form = p.top()
+ }
+ case a.Li:
+ p.framesetOK = false
+ for i := len(p.oe) - 1; i >= 0; i-- {
+ node := p.oe[i]
+ switch node.DataAtom {
+ case a.Li:
+ p.oe = p.oe[:i]
+ case a.Address, a.Div, a.P:
+ continue
+ default:
+ if !isSpecialElement(node) {
+ continue
+ }
+ }
+ break
+ }
+ p.popUntil(buttonScope, a.P)
+ p.addElement()
+ case a.Dd, a.Dt:
+ p.framesetOK = false
+ for i := len(p.oe) - 1; i >= 0; i-- {
+ node := p.oe[i]
+ switch node.DataAtom {
+ case a.Dd, a.Dt:
+ p.oe = p.oe[:i]
+ case a.Address, a.Div, a.P:
+ continue
+ default:
+ if !isSpecialElement(node) {
+ continue
+ }
+ }
+ break
+ }
+ p.popUntil(buttonScope, a.P)
+ p.addElement()
+ case a.Plaintext:
+ p.popUntil(buttonScope, a.P)
+ p.addElement()
+ case a.Button:
+ p.popUntil(defaultScope, a.Button)
+ p.reconstructActiveFormattingElements()
+ p.addElement()
+ p.framesetOK = false
+ case a.A:
+ for i := len(p.afe) - 1; i >= 0 && p.afe[i].Type != scopeMarkerNode; i-- {
+ if n := p.afe[i]; n.Type == ElementNode && n.DataAtom == a.A {
+ p.inBodyEndTagFormatting(a.A)
+ p.oe.remove(n)
+ p.afe.remove(n)
+ break
+ }
+ }
+ p.reconstructActiveFormattingElements()
+ p.addFormattingElement()
+ case a.B, a.Big, a.Code, a.Em, a.Font, a.I, a.S, a.Small, a.Strike, a.Strong, a.Tt, a.U:
+ p.reconstructActiveFormattingElements()
+ p.addFormattingElement()
+ case a.Nobr:
+ p.reconstructActiveFormattingElements()
+ if p.elementInScope(defaultScope, a.Nobr) {
+ p.inBodyEndTagFormatting(a.Nobr)
+ p.reconstructActiveFormattingElements()
+ }
+ p.addFormattingElement()
+ case a.Applet, a.Marquee, a.Object:
+ p.reconstructActiveFormattingElements()
+ p.addElement()
+ p.afe = append(p.afe, &scopeMarker)
+ p.framesetOK = false
+ case a.Table:
+ if !p.quirks {
+ p.popUntil(buttonScope, a.P)
+ }
+ p.addElement()
+ p.framesetOK = false
+ p.im = inTableIM
+ return true
+ case a.Area, a.Br, a.Embed, a.Img, a.Input, a.Keygen, a.Wbr:
+ p.reconstructActiveFormattingElements()
+ p.addElement()
+ p.oe.pop()
+ p.acknowledgeSelfClosingTag()
+ if p.tok.DataAtom == a.Input {
+ for _, t := range p.tok.Attr {
+ if t.Key == "type" {
+ if strings.ToLower(t.Val) == "hidden" {
+ // Skip setting framesetOK = false
+ return true
+ }
+ }
+ }
+ }
+ p.framesetOK = false
+ case a.Param, a.Source, a.Track:
+ p.addElement()
+ p.oe.pop()
+ p.acknowledgeSelfClosingTag()
+ case a.Hr:
+ p.popUntil(buttonScope, a.P)
+ p.addElement()
+ p.oe.pop()
+ p.acknowledgeSelfClosingTag()
+ p.framesetOK = false
+ case a.Image:
+ p.tok.DataAtom = a.Img
+ p.tok.Data = a.Img.String()
+ return false
+ case a.Isindex:
+ if p.form != nil {
+ // Ignore the token.
+ return true
+ }
+ action := ""
+ prompt := "This is a searchable index. Enter search keywords: "
+ attr := []Attribute{{Key: "name", Val: "isindex"}}
+ for _, t := range p.tok.Attr {
+ switch t.Key {
+ case "action":
+ action = t.Val
+ case "name":
+ // Ignore the attribute.
+ case "prompt":
+ prompt = t.Val
+ default:
+ attr = append(attr, t)
+ }
+ }
+ p.acknowledgeSelfClosingTag()
+ p.popUntil(buttonScope, a.P)
+ p.parseImpliedToken(StartTagToken, a.Form, a.Form.String())
+ if p.form == nil {
+ // NOTE: The 'isindex' element has been removed,
+ // and the 'template' element has not been designed to be
+ // collaborative with the index element.
+ //
+ // Ignore the token.
+ return true
+ }
+ if action != "" {
+ p.form.Attr = []Attribute{{Key: "action", Val: action}}
+ }
+ p.parseImpliedToken(StartTagToken, a.Hr, a.Hr.String())
+ p.parseImpliedToken(StartTagToken, a.Label, a.Label.String())
+ p.addText(prompt)
+ p.addChild(&Node{
+ Type: ElementNode,
+ DataAtom: a.Input,
+ Data: a.Input.String(),
+ Attr: attr,
+ })
+ p.oe.pop()
+ p.parseImpliedToken(EndTagToken, a.Label, a.Label.String())
+ p.parseImpliedToken(StartTagToken, a.Hr, a.Hr.String())
+ p.parseImpliedToken(EndTagToken, a.Form, a.Form.String())
+ case a.Textarea:
+ p.addElement()
+ p.setOriginalIM()
+ p.framesetOK = false
+ p.im = textIM
+ case a.Xmp:
+ p.popUntil(buttonScope, a.P)
+ p.reconstructActiveFormattingElements()
+ p.framesetOK = false
+ p.addElement()
+ p.setOriginalIM()
+ p.im = textIM
+ case a.Iframe:
+ p.framesetOK = false
+ p.addElement()
+ p.setOriginalIM()
+ p.im = textIM
+ case a.Noembed, a.Noscript:
+ p.addElement()
+ p.setOriginalIM()
+ p.im = textIM
+ case a.Select:
+ p.reconstructActiveFormattingElements()
+ p.addElement()
+ p.framesetOK = false
+ p.im = inSelectIM
+ return true
+ case a.Optgroup, a.Option:
+ if p.top().DataAtom == a.Option {
+ p.oe.pop()
+ }
+ p.reconstructActiveFormattingElements()
+ p.addElement()
+ case a.Rb, a.Rtc:
+ if p.elementInScope(defaultScope, a.Ruby) {
+ p.generateImpliedEndTags()
+ }
+ p.addElement()
+ case a.Rp, a.Rt:
+ if p.elementInScope(defaultScope, a.Ruby) {
+ p.generateImpliedEndTags("rtc")
+ }
+ p.addElement()
+ case a.Math, a.Svg:
+ p.reconstructActiveFormattingElements()
+ if p.tok.DataAtom == a.Math {
+ adjustAttributeNames(p.tok.Attr, mathMLAttributeAdjustments)
+ } else {
+ adjustAttributeNames(p.tok.Attr, svgAttributeAdjustments)
+ }
+ adjustForeignAttributes(p.tok.Attr)
+ p.addElement()
+ p.top().Namespace = p.tok.Data
+ if p.hasSelfClosingToken {
+ p.oe.pop()
+ p.acknowledgeSelfClosingTag()
+ }
+ return true
+ case a.Caption, a.Col, a.Colgroup, a.Frame, a.Head, a.Tbody, a.Td, a.Tfoot, a.Th, a.Thead, a.Tr:
+ // Ignore the token.
+ default:
+ p.reconstructActiveFormattingElements()
+ p.addElement()
+ }
+ case EndTagToken:
+ switch p.tok.DataAtom {
+ case a.Body:
+ if p.elementInScope(defaultScope, a.Body) {
+ p.im = afterBodyIM
+ }
+ case a.Html:
+ if p.elementInScope(defaultScope, a.Body) {
+ p.parseImpliedToken(EndTagToken, a.Body, a.Body.String())
+ return false
+ }
+ return true
+ case a.Address, a.Article, a.Aside, a.Blockquote, a.Button, a.Center, a.Details, a.Dir, a.Div, a.Dl, a.Fieldset, a.Figcaption, a.Figure, a.Footer, a.Header, a.Hgroup, a.Listing, a.Menu, a.Nav, a.Ol, a.Pre, a.Section, a.Summary, a.Ul:
+ p.popUntil(defaultScope, p.tok.DataAtom)
+ case a.Form:
+ if p.oe.contains(a.Template) {
+ i := p.indexOfElementInScope(defaultScope, a.Form)
+ if i == -1 {
+ // Ignore the token.
+ return true
+ }
+ p.generateImpliedEndTags()
+ if p.oe[i].DataAtom != a.Form {
+ // Ignore the token.
+ return true
+ }
+ p.popUntil(defaultScope, a.Form)
+ } else {
+ node := p.form
+ p.form = nil
+ i := p.indexOfElementInScope(defaultScope, a.Form)
+ if node == nil || i == -1 || p.oe[i] != node {
+ // Ignore the token.
+ return true
+ }
+ p.generateImpliedEndTags()
+ p.oe.remove(node)
+ }
+ case a.P:
+ if !p.elementInScope(buttonScope, a.P) {
+ p.parseImpliedToken(StartTagToken, a.P, a.P.String())
+ }
+ p.popUntil(buttonScope, a.P)
+ case a.Li:
+ p.popUntil(listItemScope, a.Li)
+ case a.Dd, a.Dt:
+ p.popUntil(defaultScope, p.tok.DataAtom)
+ case a.H1, a.H2, a.H3, a.H4, a.H5, a.H6:
+ p.popUntil(defaultScope, a.H1, a.H2, a.H3, a.H4, a.H5, a.H6)
+ case a.A, a.B, a.Big, a.Code, a.Em, a.Font, a.I, a.Nobr, a.S, a.Small, a.Strike, a.Strong, a.Tt, a.U:
+ p.inBodyEndTagFormatting(p.tok.DataAtom)
+ case a.Applet, a.Marquee, a.Object:
+ if p.popUntil(defaultScope, p.tok.DataAtom) {
+ p.clearActiveFormattingElements()
+ }
+ case a.Br:
+ p.tok.Type = StartTagToken
+ return false
+ case a.Template:
+ return inHeadIM(p)
+ default:
+ p.inBodyEndTagOther(p.tok.DataAtom)
+ }
+ case CommentToken:
+ p.addChild(&Node{
+ Type: CommentNode,
+ Data: p.tok.Data,
+ })
+ case ErrorToken:
+ // TODO: remove this divergence from the HTML5 spec.
+ if len(p.templateStack) > 0 {
+ p.im = inTemplateIM
+ return false
+ } else {
+ for _, e := range p.oe {
+ switch e.DataAtom {
+ case a.Dd, a.Dt, a.Li, a.Optgroup, a.Option, a.P, a.Rb, a.Rp, a.Rt, a.Rtc, a.Tbody, a.Td, a.Tfoot, a.Th,
+ a.Thead, a.Tr, a.Body, a.Html:
+ default:
+ return true
+ }
+ }
+ }
+ }
+
+ return true
+}
+
+func (p *parser) inBodyEndTagFormatting(tagAtom a.Atom) {
+ // This is the "adoption agency" algorithm, described at
+ // https://html.spec.whatwg.org/multipage/syntax.html#adoptionAgency
+
+ // TODO: this is a fairly literal line-by-line translation of that algorithm.
+ // Once the code successfully parses the comprehensive test suite, we should
+ // refactor this code to be more idiomatic.
+
+ // Steps 1-4. The outer loop.
+ for i := 0; i < 8; i++ {
+ // Step 5. Find the formatting element.
+ var formattingElement *Node
+ for j := len(p.afe) - 1; j >= 0; j-- {
+ if p.afe[j].Type == scopeMarkerNode {
+ break
+ }
+ if p.afe[j].DataAtom == tagAtom {
+ formattingElement = p.afe[j]
+ break
+ }
+ }
+ if formattingElement == nil {
+ p.inBodyEndTagOther(tagAtom)
+ return
+ }
+ feIndex := p.oe.index(formattingElement)
+ if feIndex == -1 {
+ p.afe.remove(formattingElement)
+ return
+ }
+ if !p.elementInScope(defaultScope, tagAtom) {
+ // Ignore the tag.
+ return
+ }
+
+ // Steps 9-10. Find the furthest block.
+ var furthestBlock *Node
+ for _, e := range p.oe[feIndex:] {
+ if isSpecialElement(e) {
+ furthestBlock = e
+ break
+ }
+ }
+ if furthestBlock == nil {
+ e := p.oe.pop()
+ for e != formattingElement {
+ e = p.oe.pop()
+ }
+ p.afe.remove(e)
+ return
+ }
+
+ // Steps 11-12. Find the common ancestor and bookmark node.
+ commonAncestor := p.oe[feIndex-1]
+ bookmark := p.afe.index(formattingElement)
+
+ // Step 13. The inner loop. Find the lastNode to reparent.
+ lastNode := furthestBlock
+ node := furthestBlock
+ x := p.oe.index(node)
+ // Steps 13.1-13.2
+ for j := 0; j < 3; j++ {
+ // Step 13.3.
+ x--
+ node = p.oe[x]
+ // Step 13.4 - 13.5.
+ if p.afe.index(node) == -1 {
+ p.oe.remove(node)
+ continue
+ }
+ // Step 13.6.
+ if node == formattingElement {
+ break
+ }
+ // Step 13.7.
+ clone := node.clone()
+ p.afe[p.afe.index(node)] = clone
+ p.oe[p.oe.index(node)] = clone
+ node = clone
+ // Step 13.8.
+ if lastNode == furthestBlock {
+ bookmark = p.afe.index(node) + 1
+ }
+ // Step 13.9.
+ if lastNode.Parent != nil {
+ lastNode.Parent.RemoveChild(lastNode)
+ }
+ node.AppendChild(lastNode)
+ // Step 13.10.
+ lastNode = node
+ }
+
+ // Step 14. Reparent lastNode to the common ancestor,
+ // or for misnested table nodes, to the foster parent.
+ if lastNode.Parent != nil {
+ lastNode.Parent.RemoveChild(lastNode)
+ }
+ switch commonAncestor.DataAtom {
+ case a.Table, a.Tbody, a.Tfoot, a.Thead, a.Tr:
+ p.fosterParent(lastNode)
+ default:
+ commonAncestor.AppendChild(lastNode)
+ }
+
+ // Steps 15-17. Reparent nodes from the furthest block's children
+ // to a clone of the formatting element.
+ clone := formattingElement.clone()
+ reparentChildren(clone, furthestBlock)
+ furthestBlock.AppendChild(clone)
+
+ // Step 18. Fix up the list of active formatting elements.
+ if oldLoc := p.afe.index(formattingElement); oldLoc != -1 && oldLoc < bookmark {
+ // Move the bookmark with the rest of the list.
+ bookmark--
+ }
+ p.afe.remove(formattingElement)
+ p.afe.insert(bookmark, clone)
+
+ // Step 19. Fix up the stack of open elements.
+ p.oe.remove(formattingElement)
+ p.oe.insert(p.oe.index(furthestBlock)+1, clone)
+ }
+}
+
+// inBodyEndTagOther performs the "any other end tag" algorithm for inBodyIM.
+// "Any other end tag" handling from 12.2.6.5 The rules for parsing tokens in foreign content
+// https://html.spec.whatwg.org/multipage/syntax.html#parsing-main-inforeign
+func (p *parser) inBodyEndTagOther(tagAtom a.Atom) {
+ for i := len(p.oe) - 1; i >= 0; i-- {
+ if p.oe[i].DataAtom == tagAtom {
+ p.oe = p.oe[:i]
+ break
+ }
+ if isSpecialElement(p.oe[i]) {
+ break
+ }
+ }
+}
+
+// Section 12.2.6.4.8.
+func textIM(p *parser) bool {
+ switch p.tok.Type {
+ case ErrorToken:
+ p.oe.pop()
+ case TextToken:
+ d := p.tok.Data
+ if n := p.oe.top(); n.DataAtom == a.Textarea && n.FirstChild == nil {
+ // Ignore a newline at the start of a -->
+#errors
+#document
+|
+|
+|
+| -->
+#errors
+#document
+|
+|
+|
+|
+#errors
+Line: 1 Col: 10 Unexpected start tag (textarea). Expected DOCTYPE.
+#document
+|
+|
+|
+|
+#errors
+Line: 1 Col: 9 Unexpected end tag (strong). Expected DOCTYPE.
+Line: 1 Col: 9 Unexpected end tag (strong) after the (implied) root element.
+Line: 1 Col: 13 Unexpected end tag (b) after the (implied) root element.
+Line: 1 Col: 18 Unexpected end tag (em) after the (implied) root element.
+Line: 1 Col: 22 Unexpected end tag (i) after the (implied) root element.
+Line: 1 Col: 26 Unexpected end tag (u) after the (implied) root element.
+Line: 1 Col: 35 Unexpected end tag (strike) after the (implied) root element.
+Line: 1 Col: 39 Unexpected end tag (s) after the (implied) root element.
+Line: 1 Col: 47 Unexpected end tag (blink) after the (implied) root element.
+Line: 1 Col: 52 Unexpected end tag (tt) after the (implied) root element.
+Line: 1 Col: 58 Unexpected end tag (pre) after the (implied) root element.
+Line: 1 Col: 64 Unexpected end tag (big) after the (implied) root element.
+Line: 1 Col: 72 Unexpected end tag (small) after the (implied) root element.
+Line: 1 Col: 79 Unexpected end tag (font) after the (implied) root element.
+Line: 1 Col: 88 Unexpected end tag (select) after the (implied) root element.
+Line: 1 Col: 93 Unexpected end tag (h1) after the (implied) root element.
+Line: 1 Col: 98 Unexpected end tag (h2) after the (implied) root element.
+Line: 1 Col: 103 Unexpected end tag (h3) after the (implied) root element.
+Line: 1 Col: 108 Unexpected end tag (h4) after the (implied) root element.
+Line: 1 Col: 113 Unexpected end tag (h5) after the (implied) root element.
+Line: 1 Col: 118 Unexpected end tag (h6) after the (implied) root element.
+Line: 1 Col: 125 Unexpected end tag (body) after the (implied) root element.
+Line: 1 Col: 130 Unexpected end tag (br). Treated as br element.
+Line: 1 Col: 134 End tag (a) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 140 This element (img) has no end tag.
+Line: 1 Col: 148 Unexpected end tag (title). Ignored.
+Line: 1 Col: 155 Unexpected end tag (span). Ignored.
+Line: 1 Col: 163 Unexpected end tag (style). Ignored.
+Line: 1 Col: 172 Unexpected end tag (script). Ignored.
+Line: 1 Col: 180 Unexpected end tag (table). Ignored.
+Line: 1 Col: 185 Unexpected end tag (th). Ignored.
+Line: 1 Col: 190 Unexpected end tag (td). Ignored.
+Line: 1 Col: 195 Unexpected end tag (tr). Ignored.
+Line: 1 Col: 203 This element (frame) has no end tag.
+Line: 1 Col: 210 This element (area) has no end tag.
+Line: 1 Col: 217 Unexpected end tag (link). Ignored.
+Line: 1 Col: 225 This element (param) has no end tag.
+Line: 1 Col: 230 This element (hr) has no end tag.
+Line: 1 Col: 238 This element (input) has no end tag.
+Line: 1 Col: 244 Unexpected end tag (col). Ignored.
+Line: 1 Col: 251 Unexpected end tag (base). Ignored.
+Line: 1 Col: 258 Unexpected end tag (meta). Ignored.
+Line: 1 Col: 269 This element (basefont) has no end tag.
+Line: 1 Col: 279 This element (bgsound) has no end tag.
+Line: 1 Col: 287 This element (embed) has no end tag.
+Line: 1 Col: 296 This element (spacer) has no end tag.
+Line: 1 Col: 300 Unexpected end tag (p). Ignored.
+Line: 1 Col: 305 End tag (dd) seen too early. Expected other end tag.
+Line: 1 Col: 310 End tag (dt) seen too early. Expected other end tag.
+Line: 1 Col: 320 Unexpected end tag (caption). Ignored.
+Line: 1 Col: 331 Unexpected end tag (colgroup). Ignored.
+Line: 1 Col: 339 Unexpected end tag (tbody). Ignored.
+Line: 1 Col: 347 Unexpected end tag (tfoot). Ignored.
+Line: 1 Col: 355 Unexpected end tag (thead). Ignored.
+Line: 1 Col: 365 End tag (address) seen too early. Expected other end tag.
+Line: 1 Col: 378 End tag (blockquote) seen too early. Expected other end tag.
+Line: 1 Col: 387 End tag (center) seen too early. Expected other end tag.
+Line: 1 Col: 393 Unexpected end tag (dir). Ignored.
+Line: 1 Col: 399 End tag (div) seen too early. Expected other end tag.
+Line: 1 Col: 404 End tag (dl) seen too early. Expected other end tag.
+Line: 1 Col: 415 End tag (fieldset) seen too early. Expected other end tag.
+Line: 1 Col: 425 End tag (listing) seen too early. Expected other end tag.
+Line: 1 Col: 432 End tag (menu) seen too early. Expected other end tag.
+Line: 1 Col: 437 End tag (ol) seen too early. Expected other end tag.
+Line: 1 Col: 442 End tag (ul) seen too early. Expected other end tag.
+Line: 1 Col: 447 End tag (li) seen too early. Expected other end tag.
+Line: 1 Col: 454 End tag (nobr) violates step 1, paragraph 1 of the adoption agency algorithm.
+Line: 1 Col: 460 This element (wbr) has no end tag.
+Line: 1 Col: 476 End tag (button) seen too early. Expected other end tag.
+Line: 1 Col: 486 End tag (marquee) seen too early. Expected other end tag.
+Line: 1 Col: 495 End tag (object) seen too early. Expected other end tag.
+Line: 1 Col: 513 Unexpected end tag (html). Ignored.
+Line: 1 Col: 513 Unexpected end tag (frameset). Ignored.
+Line: 1 Col: 520 Unexpected end tag (head). Ignored.
+Line: 1 Col: 529 Unexpected end tag (iframe). Ignored.
+Line: 1 Col: 537 This element (image) has no end tag.
+Line: 1 Col: 547 This element (isindex) has no end tag.
+Line: 1 Col: 557 Unexpected end tag (noembed). Ignored.
+Line: 1 Col: 568 Unexpected end tag (noframes). Ignored.
+Line: 1 Col: 579 Unexpected end tag (noscript). Ignored.
+Line: 1 Col: 590 Unexpected end tag (optgroup). Ignored.
+Line: 1 Col: 599 Unexpected end tag (option). Ignored.
+Line: 1 Col: 611 Unexpected end tag (plaintext). Ignored.
+Line: 1 Col: 622 Unexpected end tag (textarea). Ignored.
+#document
+|
+|
+|
+|
+|
+
+#data
+