youtubebeat/vendor/github.com/elastic/beats/journalbeat/reader/journal.go

260 lines
6.6 KiB
Go

// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//+build linux,cgo
package reader
import (
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"github.com/coreos/go-systemd/sdjournal"
"github.com/pkg/errors"
"github.com/elastic/beats/journalbeat/checkpoint"
"github.com/elastic/beats/journalbeat/cmd/instance"
"github.com/elastic/beats/libbeat/beat"
"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/logp"
)
// Reader reads entries from journal(s).
type Reader struct {
journal *sdjournal.Journal
config Config
done chan struct{}
logger *logp.Logger
backoff *common.Backoff
}
// New creates a new journal reader and moves the FP to the configured position.
func New(c Config, done chan struct{}, state checkpoint.JournalState, logger *logp.Logger) (*Reader, error) {
f, err := os.Stat(c.Path)
if err != nil {
return nil, errors.Wrap(err, "failed to open file")
}
var j *sdjournal.Journal
if f.IsDir() {
j, err = sdjournal.NewJournalFromDir(c.Path)
if err != nil {
return nil, errors.Wrap(err, "failed to open journal directory")
}
} else {
j, err = sdjournal.NewJournalFromFiles(c.Path)
if err != nil {
return nil, errors.Wrap(err, "failed to open journal file")
}
}
l := logger.With("path", c.Path)
l.Debug("New journal is opened for reading")
return newReader(l, done, c, j, state)
}
// NewLocal creates a reader to read form the local journal and moves the FP
// to the configured position.
func NewLocal(c Config, done chan struct{}, state checkpoint.JournalState, logger *logp.Logger) (*Reader, error) {
j, err := sdjournal.NewJournal()
if err != nil {
return nil, errors.Wrap(err, "failed to open local journal")
}
l := logger.With("path", "local")
l.Debug("New local journal is opened for reading")
return newReader(l, done, c, j, state)
}
func newReader(logger *logp.Logger, done chan struct{}, c Config, journal *sdjournal.Journal, state checkpoint.JournalState) (*Reader, error) {
err := setupMatches(journal, c.Matches)
if err != nil {
return nil, err
}
r := &Reader{
journal: journal,
config: c,
done: done,
logger: logger,
backoff: common.NewBackoff(done, c.Backoff, c.MaxBackoff),
}
r.seek(state.Cursor)
instance.AddJournalToMonitor(c.Path, journal)
return r, nil
}
func setupMatches(j *sdjournal.Journal, matches []string) error {
for _, m := range matches {
elems := strings.Split(m, "=")
if len(elems) != 2 {
return fmt.Errorf("invalid match format: %s", m)
}
var p string
for journalKey, eventField := range journaldEventFields {
if elems[0] == eventField.name {
p = journalKey + "=" + elems[1]
}
}
// pass custom fields as is
if p == "" {
p = m
}
logp.Debug("journal", "Added matcher expression: %s", p)
err := j.AddMatch(p)
if err != nil {
return fmt.Errorf("error adding match to journal %v", err)
}
err = j.AddDisjunction()
if err != nil {
return fmt.Errorf("error adding disjunction to journal: %v", err)
}
}
return nil
}
// seek seeks to the position determined by the coniguration and cursor state.
func (r *Reader) seek(cursor string) {
if r.config.Seek == "cursor" {
if cursor == "" {
r.journal.SeekHead()
r.logger.Debug("Seeking method set to cursor, but no state is saved for reader. Starting to read from the beginning")
return
}
r.journal.SeekCursor(cursor)
_, err := r.journal.Next()
if err != nil {
r.logger.Error("Error while seeking to cursor")
}
r.logger.Debug("Seeked to position defined in cursor")
} else if r.config.Seek == "tail" {
r.journal.SeekTail()
r.logger.Debug("Tailing the journal file")
} else if r.config.Seek == "head" {
r.journal.SeekHead()
r.logger.Debug("Reading from the beginning of the journal file")
}
}
// Next waits until a new event shows up and returns it.
// It blocks until an event is returned or an error occurs.
func (r *Reader) Next() (*beat.Event, error) {
for {
select {
case <-r.done:
return nil, nil
default:
event, err := r.readEvent()
if err != nil {
return nil, err
}
if event == nil {
r.backoff.Wait()
continue
}
r.backoff.Reset()
return event, nil
}
}
}
func (r *Reader) readEvent() (*beat.Event, error) {
n, err := r.journal.Next()
if err != nil && err != io.EOF {
return nil, err
}
for n == 1 {
entry, err := r.journal.GetEntry()
if err != nil {
return nil, err
}
event := r.toEvent(entry)
return event, nil
}
return nil, nil
}
// toEvent creates a beat.Event from journal entries.
func (r *Reader) toEvent(entry *sdjournal.JournalEntry) *beat.Event {
fields := common.MapStr{}
custom := common.MapStr{}
for entryKey, v := range entry.Fields {
if fieldConversionInfo, ok := journaldEventFields[entryKey]; !ok {
normalized := strings.ToLower(strings.TrimLeft(entryKey, "_"))
custom.Put(normalized, v)
} else if !fieldConversionInfo.dropped {
value := r.convertNamedField(fieldConversionInfo, v)
fields.Put(fieldConversionInfo.name, value)
}
}
if len(custom) != 0 {
fields["custom"] = custom
}
state := checkpoint.JournalState{
Path: r.config.Path,
Cursor: entry.Cursor,
RealtimeTimestamp: entry.RealtimeTimestamp,
MonotonicTimestamp: entry.MonotonicTimestamp,
}
fields["read_timestamp"] = time.Now()
receivedByJournal := time.Unix(0, int64(entry.RealtimeTimestamp)*1000)
event := beat.Event{
Timestamp: receivedByJournal,
Fields: fields,
Private: state,
}
return &event
}
func (r *Reader) convertNamedField(fc fieldConversion, value string) interface{} {
if fc.isInteger {
v, err := strconv.ParseInt(value, 10, 64)
if err != nil {
r.logger.Debugf("Failed to convert field: %s \"%v\" to int: %v", fc.name, value, err)
return value
}
return v
}
return value
}
// Close closes the underlying journal reader.
func (r *Reader) Close() {
instance.StopMonitoringJournal(r.config.Path)
r.journal.Close()
}