119 lines
2.8 KiB
Go
119 lines
2.8 KiB
Go
|
// Licensed to Elasticsearch B.V. under one or more contributor
|
||
|
|
// license agreements. See the NOTICE file distributed with
|
||
|
|
// this work for additional information regarding copyright
|
||
|
|
// ownership. Elasticsearch B.V. licenses this file to you under
|
||
|
|
// the Apache License, Version 2.0 (the "License"); you may
|
||
|
|
// not use this file except in compliance with the License.
|
||
|
|
// You may obtain a copy of the License at
|
||
|
|
//
|
||
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
//
|
||
|
|
// Unless required by applicable law or agreed to in writing,
|
||
|
|
// software distributed under the License is distributed on an
|
||
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||
|
|
// KIND, either express or implied. See the License for the
|
||
|
|
// specific language governing permissions and limitations
|
||
|
|
// under the License.
|
||
|
|
|
||
|
|
package redis
|
||
|
|
|
||
|
|
import (
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/garyburd/redigo/redis"
|
||
|
|
|
||
|
|
"github.com/elastic/beats/libbeat/common"
|
||
|
|
"github.com/elastic/beats/libbeat/publisher"
|
||
|
|
)
|
||
|
|
|
||
|
|
type backoffClient struct {
|
||
|
|
client *client
|
||
|
|
|
||
|
|
reason failReason
|
||
|
|
|
||
|
|
done chan struct{}
|
||
|
|
backoff *common.Backoff
|
||
|
|
}
|
||
|
|
|
||
|
|
// failReason is used to track the cause of an error.
|
||
|
|
// The redis client forces us to reconnect on any error (even for redis
|
||
|
|
// internal errors). The backoff timer must not be reset on a successful
|
||
|
|
// reconnect after publishing failed with a redis internal
|
||
|
|
// error (e.g. OutOfMemory), so we can still guarantee the backoff duration
|
||
|
|
// increases exponentially.
|
||
|
|
type failReason uint8
|
||
|
|
|
||
|
|
const (
|
||
|
|
failNone failReason = iota
|
||
|
|
failRedis
|
||
|
|
failOther
|
||
|
|
)
|
||
|
|
|
||
|
|
func newBackoffClient(client *client, init, max time.Duration) *backoffClient {
|
||
|
|
done := make(chan struct{})
|
||
|
|
backoff := common.NewBackoff(done, init, max)
|
||
|
|
return &backoffClient{
|
||
|
|
client: client,
|
||
|
|
done: done,
|
||
|
|
backoff: backoff,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (b *backoffClient) Connect() error {
|
||
|
|
err := b.client.Connect()
|
||
|
|
if err != nil {
|
||
|
|
// give the client a chance to promote an internal error to a network error.
|
||
|
|
b.updateFailReason(err)
|
||
|
|
b.backoff.Wait()
|
||
|
|
} else if b.reason != failRedis { // Only reset backoff duration if failure was due to IO errors.
|
||
|
|
b.resetFail()
|
||
|
|
}
|
||
|
|
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
func (b *backoffClient) Close() error {
|
||
|
|
err := b.client.Close()
|
||
|
|
close(b.done)
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
func (b *backoffClient) Publish(batch publisher.Batch) error {
|
||
|
|
err := b.client.Publish(batch)
|
||
|
|
if err != nil {
|
||
|
|
b.client.Close()
|
||
|
|
b.updateFailReason(err)
|
||
|
|
b.backoff.Wait()
|
||
|
|
} else {
|
||
|
|
b.resetFail()
|
||
|
|
}
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
func (b *backoffClient) updateFailReason(err error) {
|
||
|
|
if b.reason == failRedis {
|
||
|
|
// we only allow 'Publish' to recover from an redis internal error
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
if err == nil {
|
||
|
|
b.reason = failNone
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
if _, ok := err.(redis.Error); ok {
|
||
|
|
b.reason = failRedis
|
||
|
|
} else {
|
||
|
|
b.reason = failOther
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (b *backoffClient) resetFail() {
|
||
|
|
b.reason = failNone
|
||
|
|
b.backoff.Reset()
|
||
|
|
}
|
||
|
|
|
||
|
|
func (b *backoffClient) String() string {
|
||
|
|
return b.client.String()
|
||
|
|
}
|