youtubebeat/vendor/github.com/elastic/beats/journalbeat/reader/journal.go

// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

//+build linux,cgo

package reader

import (
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/coreos/go-systemd/sdjournal"
	"github.com/pkg/errors"

	"github.com/elastic/beats/journalbeat/checkpoint"
	"github.com/elastic/beats/journalbeat/cmd/instance"
	"github.com/elastic/beats/libbeat/beat"
	"github.com/elastic/beats/libbeat/common"
	"github.com/elastic/beats/libbeat/logp"
)

// Reader reads entries from journal(s).
type Reader struct {
	journal *sdjournal.Journal
	config  Config
	done    chan struct{}
	logger  *logp.Logger
	backoff *common.Backoff
}

// New creates a new journal reader and moves the FP to the configured position.
func New(c Config, done chan struct{}, state checkpoint.JournalState, logger *logp.Logger) (*Reader, error) {
	f, err := os.Stat(c.Path)
	if err != nil {
		return nil, errors.Wrap(err, "failed to open file")
	}

	var j *sdjournal.Journal
	if f.IsDir() {
		j, err = sdjournal.NewJournalFromDir(c.Path)
		if err != nil {
			return nil, errors.Wrap(err, "failed to open journal directory")
		}
	} else {
		j, err = sdjournal.NewJournalFromFiles(c.Path)
		if err != nil {
			return nil, errors.Wrap(err, "failed to open journal file")
		}
	}

	l := logger.With("path", c.Path)
	l.Debug("New journal is opened for reading")

	return newReader(l, done, c, j, state)
}

// NewLocal creates a reader to read form the local journal and moves the FP
// to the configured position.
func NewLocal(c Config, done chan struct{}, state checkpoint.JournalState, logger *logp.Logger) (*Reader, error) {
	j, err := sdjournal.NewJournal()
	if err != nil {
		return nil, errors.Wrap(err, "failed to open local journal")
	}

	l := logger.With("path", "local")
	l.Debug("New local journal is opened for reading")

	return newReader(l, done, c, j, state)
}

func newReader(logger *logp.Logger, done chan struct{}, c Config, journal *sdjournal.Journal, state checkpoint.JournalState) (*Reader, error) {
	err := setupMatches(journal, c.Matches)
	if err != nil {
		return nil, err
	}

	r := &Reader{
		journal: journal,
		config:  c,
		done:    done,
		logger:  logger,
		backoff: common.NewBackoff(done, c.Backoff, c.MaxBackoff),
	}
	r.seek(state.Cursor)

	instance.AddJournalToMonitor(c.Path, journal)

	return r, nil
}

func setupMatches(j *sdjournal.Journal, matches []string) error {
	for _, m := range matches {
		elems := strings.Split(m, "=")
		if len(elems) != 2 {
			return fmt.Errorf("invalid match format: %s", m)
		}

		var p string
		for journalKey, eventField := range journaldEventFields {
			if elems[0] == eventField.name {
				p = journalKey + "=" + elems[1]
			}
		}

		// pass custom fields as is
		if p == "" {
			p = m
		}

		logp.Debug("journal", "Added matcher expression: %s", p)

		err := j.AddMatch(p)
		if err != nil {
			return fmt.Errorf("error adding match to journal %v", err)
		}

		err = j.AddDisjunction()
		if err != nil {
			return fmt.Errorf("error adding disjunction to journal: %v", err)
		}
	}
	return nil
}

// seek seeks to the position determined by the coniguration and cursor state.
func (r *Reader) seek(cursor string) {
	if r.config.Seek == "cursor" {
		if cursor == "" {
			r.journal.SeekHead()
			r.logger.Debug("Seeking method set to cursor, but no state is saved for reader. Starting to read from the beginning")
			return
		}
		r.journal.SeekCursor(cursor)
		_, err := r.journal.Next()
		if err != nil {
			r.logger.Error("Error while seeking to cursor")
		}
		r.logger.Debug("Seeked to position defined in cursor")
	} else if r.config.Seek == "tail" {
		r.journal.SeekTail()
		r.logger.Debug("Tailing the journal file")
	} else if r.config.Seek == "head" {
		r.journal.SeekHead()
		r.logger.Debug("Reading from the beginning of the journal file")
	}
}

// Next waits until a new event shows up and returns it.
// It blocks until an event is returned or an error occurs.
func (r *Reader) Next() (*beat.Event, error) {
	for {
		select {
		case <-r.done:
			return nil, nil
		default:
			event, err := r.readEvent()
			if err != nil {
				return nil, err
			}

			if event == nil {
				r.backoff.Wait()
				continue
			}

			r.backoff.Reset()
			return event, nil
		}
	}
}

func (r *Reader) readEvent() (*beat.Event, error) {
	n, err := r.journal.Next()
	if err != nil && err != io.EOF {
		return nil, err
	}

	for n == 1 {
		entry, err := r.journal.GetEntry()
		if err != nil {
			return nil, err
		}
		event := r.toEvent(entry)
		return event, nil
	}
	return nil, nil
}

// toEvent creates a beat.Event from journal entries.
func (r *Reader) toEvent(entry *sdjournal.JournalEntry) *beat.Event {
	fields := common.MapStr{}
	custom := common.MapStr{}

	for entryKey, v := range entry.Fields {
		if fieldConversionInfo, ok := journaldEventFields[entryKey]; !ok {
			normalized := strings.ToLower(strings.TrimLeft(entryKey, "_"))
			custom.Put(normalized, v)
		} else if !fieldConversionInfo.dropped {
			value := r.convertNamedField(fieldConversionInfo, v)
			fields.Put(fieldConversionInfo.name, value)
		}
	}

	if len(custom) != 0 {
		fields["custom"] = custom
	}

	state := checkpoint.JournalState{
		Path:               r.config.Path,
		Cursor:             entry.Cursor,
		RealtimeTimestamp:  entry.RealtimeTimestamp,
		MonotonicTimestamp: entry.MonotonicTimestamp,
	}

	fields["read_timestamp"] = time.Now()
	receivedByJournal := time.Unix(0, int64(entry.RealtimeTimestamp)*1000)

	event := beat.Event{
		Timestamp: receivedByJournal,
		Fields:    fields,
		Private:   state,
	}
	return &event
}

func (r *Reader) convertNamedField(fc fieldConversion, value string) interface{} {
	if fc.isInteger {
		v, err := strconv.ParseInt(value, 10, 64)
		if err != nil {
			r.logger.Debugf("Failed to convert field: %s \"%v\" to int: %v", fc.name, value, err)
			return value
		}
		return v
	}
	return value
}

// Close closes the underlying journal reader.
func (r *Reader) Close() {
	instance.StopMonitoringJournal(r.config.Path)
	r.journal.Close()
}