youtubebeat/vendor/github.com/elastic/beats/filebeat/input/log/log.go

187 lines
4.6 KiB
Go

// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package log
import (
"io"
"os"
"time"
"github.com/elastic/beats/filebeat/harvester"
"github.com/elastic/beats/filebeat/input/file"
"github.com/elastic/beats/libbeat/logp"
)
// Log contains all log related data
type Log struct {
fs harvester.Source
offset int64
config LogConfig
lastTimeRead time.Time
backoff time.Duration
done chan struct{}
}
// NewLog creates a new log instance to read log sources
func NewLog(
fs harvester.Source,
config LogConfig,
) (*Log, error) {
var offset int64
if seeker, ok := fs.(io.Seeker); ok {
var err error
offset, err = seeker.Seek(0, os.SEEK_CUR)
if err != nil {
return nil, err
}
}
return &Log{
fs: fs,
offset: offset,
config: config,
lastTimeRead: time.Now(),
backoff: config.Backoff,
done: make(chan struct{}),
}, nil
}
// Read reads from the reader and updates the offset
// The total number of bytes read is returned.
func (f *Log) Read(buf []byte) (int, error) {
totalN := 0
for {
select {
case <-f.done:
return 0, ErrClosed
default:
}
n, err := f.fs.Read(buf)
if n > 0 {
f.offset += int64(n)
f.lastTimeRead = time.Now()
}
totalN += n
// Read from source completed without error
// Either end reached or buffer full
if err == nil {
// reset backoff for next read
f.backoff = f.config.Backoff
return totalN, nil
}
// Move buffer forward for next read
buf = buf[n:]
// Checks if an error happened or buffer is full
// If buffer is full, cannot continue reading.
// Can happen if n == bufferSize + io.EOF error
err = f.errorChecks(err)
if err != nil || len(buf) == 0 {
return totalN, err
}
logp.Debug("harvester", "End of file reached: %s; Backoff now.", f.fs.Name())
f.wait()
}
}
// errorChecks checks how the given error should be handled based on the config options
func (f *Log) errorChecks(err error) error {
if err != io.EOF {
logp.Err("Unexpected state reading from %s; error: %s", f.fs.Name(), err)
return err
}
// Stdin is not continuable
if !f.fs.Continuable() {
logp.Debug("harvester", "Source is not continuable: %s", f.fs.Name())
return err
}
if err == io.EOF && f.config.CloseEOF {
return err
}
// Refetch fileinfo to check if the file was truncated or disappeared.
// Errors if the file was removed/rotated after reading and before
// calling the stat function
info, statErr := f.fs.Stat()
if statErr != nil {
logp.Err("Unexpected error reading from %s; error: %s", f.fs.Name(), statErr)
return statErr
}
// check if file was truncated
if info.Size() < f.offset {
logp.Debug("harvester",
"File was truncated as offset (%d) > size (%d): %s", f.offset, info.Size(), f.fs.Name())
return ErrFileTruncate
}
// Check file wasn't read for longer then CloseInactive
age := time.Since(f.lastTimeRead)
if age > f.config.CloseInactive {
return ErrInactive
}
if f.config.CloseRenamed {
// Check if the file can still be found under the same path
if !file.IsSameFile(f.fs.Name(), info) {
return ErrRenamed
}
}
if f.config.CloseRemoved {
// Check if the file name exists. See https://github.com/elastic/filebeat/issues/93
_, statErr := os.Stat(f.fs.Name())
// Error means file does not exist.
if statErr != nil {
return ErrRemoved
}
}
return nil
}
func (f *Log) wait() {
// Wait before trying to read file again. File reached EOF.
select {
case <-f.done:
return
case <-time.After(f.backoff):
}
// Increment backoff up to maxBackoff
if f.backoff < f.config.MaxBackoff {
f.backoff = f.backoff * time.Duration(f.config.BackoffFactor)
if f.backoff > f.config.MaxBackoff {
f.backoff = f.config.MaxBackoff
}
}
}
// Close closes the done channel but no th the file handler
func (f *Log) Close() {
close(f.done)
// Note: File reader is not closed here because that leads to race conditions
}