// Licensed to Elasticsearch B.V. under one or more contributor // license agreements. See the NOTICE file distributed with // this work for additional information regarding copyright // ownership. Elasticsearch B.V. licenses this file to you under // the Apache License, Version 2.0 (the "License"); you may // not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. package kafka import ( "errors" "sync" "time" "github.com/Shopify/sarama" gometrics "github.com/rcrowley/go-metrics" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/outputs" "github.com/elastic/beats/libbeat/outputs/codec" "github.com/elastic/beats/libbeat/outputs/outil" ) const ( defaultWaitRetry = 1 * time.Second // NOTE: maxWaitRetry has no effect on mode, as logstash client currently does // not return ErrTempBulkFailure defaultMaxWaitRetry = 60 * time.Second ) var kafkaMetricsOnce sync.Once var kafkaMetricsRegistryInstance gometrics.Registry var debugf = logp.MakeDebug("kafka") var ( errNoTopicSet = errors.New("No topic configured") errNoHosts = errors.New("No hosts configured") ) func init() { sarama.Logger = kafkaLogger{} reg := gometrics.NewPrefixedRegistry("libbeat.kafka.") // Note: registers /debug/metrics handler for displaying all expvar counters // TODO: enable //exp.Exp(reg) kafkaMetricsRegistryInstance = reg outputs.RegisterType("kafka", makeKafka) } func kafkaMetricsRegistry() gometrics.Registry { return kafkaMetricsRegistryInstance } func makeKafka( beat beat.Info, observer outputs.Observer, cfg *common.Config, ) (outputs.Group, error) { debugf("initialize kafka output") config := defaultConfig() if err := cfg.Unpack(&config); err != nil { return outputs.Fail(err) } topic, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{ Key: "topic", MultiKey: "topics", EnableSingleOnly: true, FailEmpty: true, }) if err != nil { return outputs.Fail(err) } libCfg, err := newSaramaConfig(&config) if err != nil { return outputs.Fail(err) } hosts, err := outputs.ReadHostList(cfg) if err != nil { return outputs.Fail(err) } codec, err := codec.CreateEncoder(beat, config.Codec) if err != nil { return outputs.Fail(err) } client, err := newKafkaClient(observer, hosts, beat.Beat, config.Key, topic, codec, libCfg) if err != nil { return outputs.Fail(err) } retry := 0 if config.MaxRetries < 0 { retry = -1 } return outputs.Success(config.BulkMaxSize, retry, client) }