917 lines
24 KiB
Go
917 lines
24 KiB
Go
// Licensed to Elasticsearch B.V. under one or more contributor
|
|
// license agreements. See the NOTICE file distributed with
|
|
// this work for additional information regarding copyright
|
|
// ownership. Elasticsearch B.V. licenses this file to you under
|
|
// the Apache License, Version 2.0 (the "License"); you may
|
|
// not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
package instance
|
|
|
|
import (
|
|
"context"
|
|
cryptRand "crypto/rand"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"math"
|
|
"math/big"
|
|
"math/rand"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/gofrs/uuid"
|
|
"go.uber.org/zap"
|
|
|
|
errw "github.com/pkg/errors"
|
|
|
|
"github.com/elastic/go-sysinfo"
|
|
"github.com/elastic/go-sysinfo/types"
|
|
ucfg "github.com/elastic/go-ucfg"
|
|
|
|
"github.com/elastic/beats/libbeat/api"
|
|
"github.com/elastic/beats/libbeat/asset"
|
|
"github.com/elastic/beats/libbeat/beat"
|
|
"github.com/elastic/beats/libbeat/cfgfile"
|
|
"github.com/elastic/beats/libbeat/cloudid"
|
|
"github.com/elastic/beats/libbeat/common"
|
|
"github.com/elastic/beats/libbeat/common/cfgwarn"
|
|
"github.com/elastic/beats/libbeat/common/file"
|
|
"github.com/elastic/beats/libbeat/common/reload"
|
|
"github.com/elastic/beats/libbeat/common/seccomp"
|
|
"github.com/elastic/beats/libbeat/dashboards"
|
|
"github.com/elastic/beats/libbeat/keystore"
|
|
"github.com/elastic/beats/libbeat/logp"
|
|
"github.com/elastic/beats/libbeat/logp/configure"
|
|
"github.com/elastic/beats/libbeat/management"
|
|
"github.com/elastic/beats/libbeat/metric/system/host"
|
|
"github.com/elastic/beats/libbeat/monitoring"
|
|
"github.com/elastic/beats/libbeat/monitoring/report"
|
|
"github.com/elastic/beats/libbeat/monitoring/report/log"
|
|
"github.com/elastic/beats/libbeat/outputs/elasticsearch"
|
|
"github.com/elastic/beats/libbeat/paths"
|
|
"github.com/elastic/beats/libbeat/plugin"
|
|
"github.com/elastic/beats/libbeat/publisher/pipeline"
|
|
svc "github.com/elastic/beats/libbeat/service"
|
|
"github.com/elastic/beats/libbeat/template"
|
|
"github.com/elastic/beats/libbeat/version"
|
|
|
|
// Register publisher pipeline modules
|
|
_ "github.com/elastic/beats/libbeat/publisher/includes"
|
|
|
|
// Register default processors.
|
|
_ "github.com/elastic/beats/libbeat/processors/actions"
|
|
_ "github.com/elastic/beats/libbeat/processors/add_cloud_metadata"
|
|
_ "github.com/elastic/beats/libbeat/processors/add_docker_metadata"
|
|
_ "github.com/elastic/beats/libbeat/processors/add_host_metadata"
|
|
_ "github.com/elastic/beats/libbeat/processors/add_kubernetes_metadata"
|
|
_ "github.com/elastic/beats/libbeat/processors/add_locale"
|
|
_ "github.com/elastic/beats/libbeat/processors/add_process_metadata"
|
|
_ "github.com/elastic/beats/libbeat/processors/dissect"
|
|
_ "github.com/elastic/beats/libbeat/processors/dns"
|
|
|
|
// Register autodiscover providers
|
|
_ "github.com/elastic/beats/libbeat/autodiscover/providers/docker"
|
|
_ "github.com/elastic/beats/libbeat/autodiscover/providers/jolokia"
|
|
_ "github.com/elastic/beats/libbeat/autodiscover/providers/kubernetes"
|
|
|
|
// Register default monitoring reporting
|
|
_ "github.com/elastic/beats/libbeat/monitoring/report/elasticsearch"
|
|
)
|
|
|
|
// Beat provides the runnable and configurable instance of a beat.
|
|
type Beat struct {
|
|
beat.Beat
|
|
|
|
Config beatConfig
|
|
RawConfig *common.Config // Raw config that can be unpacked to get Beat specific config data.
|
|
keystore keystore.Keystore
|
|
}
|
|
|
|
type beatConfig struct {
|
|
beat.BeatConfig `config:",inline"`
|
|
|
|
// instance internal configs
|
|
|
|
// beat top-level settings
|
|
Name string `config:"name"`
|
|
MaxProcs int `config:"max_procs"`
|
|
Seccomp *common.Config `config:"seccomp"`
|
|
|
|
// beat internal components configurations
|
|
HTTP *common.Config `config:"http"`
|
|
Path paths.Path `config:"path"`
|
|
Logging *common.Config `config:"logging"`
|
|
MetricLogging *common.Config `config:"logging.metrics"`
|
|
Keystore *common.Config `config:"keystore"`
|
|
|
|
// output/publishing related configurations
|
|
Pipeline pipeline.Config `config:",inline"`
|
|
Monitoring *common.Config `config:"xpack.monitoring"`
|
|
|
|
// central managmenet settings
|
|
Management *common.Config `config:"management"`
|
|
|
|
// elastic stack 'setup' configurations
|
|
Dashboards *common.Config `config:"setup.dashboards"`
|
|
Template *common.Config `config:"setup.template"`
|
|
Kibana *common.Config `config:"setup.kibana"`
|
|
}
|
|
|
|
var (
|
|
printVersion bool
|
|
setup bool
|
|
)
|
|
|
|
var debugf = logp.MakeDebug("beat")
|
|
|
|
func init() {
|
|
initRand()
|
|
|
|
flag.BoolVar(&printVersion, "version", false, "Print the version and exit")
|
|
flag.BoolVar(&setup, "setup", false, "Load sample Kibana dashboards and setup Machine Learning")
|
|
}
|
|
|
|
// initRand initializes the runtime random number generator seed using
|
|
// global, shared cryptographically strong pseudo random number generator.
|
|
//
|
|
// On linux Reader might use getrandom(2) or /udev/random. On windows systems
|
|
// CryptGenRandom is used.
|
|
func initRand() {
|
|
n, err := cryptRand.Int(cryptRand.Reader, big.NewInt(math.MaxInt64))
|
|
var seed int64
|
|
if err != nil {
|
|
// fallback to current timestamp
|
|
seed = time.Now().UnixNano()
|
|
} else {
|
|
seed = n.Int64()
|
|
}
|
|
rand.Seed(seed)
|
|
}
|
|
|
|
// Run initializes and runs a Beater implementation. name is the name of the
|
|
// Beat (e.g. packetbeat or metricbeat). version is version number of the Beater
|
|
// implementation. bt is the `Creator` callback for creating a new beater
|
|
// instance.
|
|
// XXX Move this as a *Beat method?
|
|
func Run(settings Settings, bt beat.Creator) error {
|
|
name := settings.Name
|
|
idxPrefix := settings.IndexPrefix
|
|
version := settings.Version
|
|
|
|
return handleError(func() error {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
logp.NewLogger(name).Fatalw("Failed due to panic.",
|
|
"panic", r, zap.Stack("stack"))
|
|
}
|
|
}()
|
|
b, err := NewBeat(name, idxPrefix, version)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Add basic info
|
|
registry := monitoring.GetNamespace("info").GetRegistry()
|
|
monitoring.NewString(registry, "version").Set(b.Info.Version)
|
|
monitoring.NewString(registry, "beat").Set(b.Info.Beat)
|
|
monitoring.NewString(registry, "name").Set(b.Info.Name)
|
|
monitoring.NewString(registry, "uuid").Set(b.Info.UUID.String())
|
|
monitoring.NewString(registry, "hostname").Set(b.Info.Hostname)
|
|
|
|
// Add additional info to state registry. This is also reported to monitoring
|
|
stateRegistry := monitoring.GetNamespace("state").GetRegistry()
|
|
serviceRegistry := stateRegistry.NewRegistry("service")
|
|
monitoring.NewString(serviceRegistry, "version").Set(b.Info.Version)
|
|
monitoring.NewString(serviceRegistry, "name").Set(b.Info.Beat)
|
|
monitoring.NewString(serviceRegistry, "id").Set(b.Info.UUID.String())
|
|
beatRegistry := stateRegistry.NewRegistry("beat")
|
|
monitoring.NewString(beatRegistry, "name").Set(b.Info.Name)
|
|
monitoring.NewFunc(stateRegistry, "host", host.ReportInfo, monitoring.Report)
|
|
|
|
return b.launch(settings, bt)
|
|
}())
|
|
}
|
|
|
|
// NewBeat creates a new beat instance
|
|
func NewBeat(name, indexPrefix, v string) (*Beat, error) {
|
|
if v == "" {
|
|
v = version.GetDefaultVersion()
|
|
}
|
|
if indexPrefix == "" {
|
|
indexPrefix = name
|
|
}
|
|
|
|
hostname, err := os.Hostname()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
fields, err := asset.GetFields(name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
id, err := uuid.NewV4()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
b := beat.Beat{
|
|
Info: beat.Info{
|
|
Beat: name,
|
|
IndexPrefix: indexPrefix,
|
|
Version: v,
|
|
Name: hostname,
|
|
Hostname: hostname,
|
|
UUID: id,
|
|
},
|
|
Fields: fields,
|
|
}
|
|
|
|
return &Beat{Beat: b}, nil
|
|
}
|
|
|
|
// InitWithSettings does initialization of things common to all actions (read confs, flags)
|
|
func (b *Beat) InitWithSettings(settings Settings) error {
|
|
err := b.handleFlags()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := plugin.Initialize(); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := b.configure(settings); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Init does initialization of things common to all actions (read confs, flags)
|
|
//
|
|
// Deprecated: use InitWithSettings
|
|
func (b *Beat) Init() error {
|
|
return b.InitWithSettings(Settings{})
|
|
}
|
|
|
|
// BeatConfig returns config section for this beat
|
|
func (b *Beat) BeatConfig() (*common.Config, error) {
|
|
configName := strings.ToLower(b.Info.Beat)
|
|
if b.RawConfig.HasField(configName) {
|
|
sub, err := b.RawConfig.Child(configName, -1)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return sub, nil
|
|
}
|
|
|
|
return common.NewConfig(), nil
|
|
}
|
|
|
|
// Keystore return the configured keystore for this beat
|
|
func (b *Beat) Keystore() keystore.Keystore {
|
|
return b.keystore
|
|
}
|
|
|
|
// create and return the beater, this method also initializes all needed items,
|
|
// including template registering, publisher, xpack monitoring
|
|
func (b *Beat) createBeater(bt beat.Creator) (beat.Beater, error) {
|
|
sub, err := b.BeatConfig()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
logSystemInfo(b.Info)
|
|
logp.Info("Setup Beat: %s; Version: %s", b.Info.Beat, b.Info.Version)
|
|
|
|
err = b.registerTemplateLoading()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
reg := monitoring.Default.GetRegistry("libbeat")
|
|
if reg == nil {
|
|
reg = monitoring.Default.NewRegistry("libbeat")
|
|
}
|
|
|
|
err = setupMetrics(b.Info.Beat)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Report central management state
|
|
mgmt := monitoring.GetNamespace("state").GetRegistry().NewRegistry("management")
|
|
monitoring.NewBool(mgmt, "enabled").Set(b.ConfigManager.Enabled())
|
|
|
|
debugf("Initializing output plugins")
|
|
|
|
outputEnabled := b.Config.Output.IsSet() && b.Config.Output.Config().Enabled()
|
|
if !outputEnabled {
|
|
if b.ConfigManager.Enabled() {
|
|
logp.Info("Output is configured through Central Management")
|
|
} else {
|
|
msg := "No outputs are defined. Please define one under the output section."
|
|
logp.Info(msg)
|
|
return nil, errors.New(msg)
|
|
}
|
|
}
|
|
|
|
pipeline, err := pipeline.Load(b.Info,
|
|
pipeline.Monitors{
|
|
Metrics: reg,
|
|
Telemetry: monitoring.GetNamespace("state").GetRegistry(),
|
|
Logger: logp.L().Named("publisher"),
|
|
},
|
|
b.Config.Pipeline,
|
|
b.Config.Output)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error initializing publisher: %+v", err)
|
|
}
|
|
|
|
reload.Register.MustRegister("output", pipeline.OutputReloader())
|
|
|
|
// TODO: some beats race on shutdown with publisher.Stop -> do not call Stop yet,
|
|
// but refine publisher to disconnect clients on stop automatically
|
|
// defer pipeline.Close()
|
|
|
|
b.Publisher = pipeline
|
|
beater, err := bt(&b.Beat, sub)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return beater, nil
|
|
}
|
|
|
|
func (b *Beat) launch(settings Settings, bt beat.Creator) error {
|
|
defer logp.Sync()
|
|
defer logp.Info("%s stopped.", b.Info.Beat)
|
|
|
|
err := b.InitWithSettings(settings)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
svc.BeforeRun()
|
|
defer svc.Cleanup()
|
|
|
|
if err = seccomp.LoadFilter(b.Config.Seccomp); err != nil {
|
|
return err
|
|
}
|
|
|
|
beater, err := b.createBeater(bt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if b.Config.Monitoring.Enabled() {
|
|
settings := report.Settings{
|
|
DefaultUsername: settings.Monitoring.DefaultUsername,
|
|
}
|
|
reporter, err := report.New(b.Info, settings, b.Config.Monitoring, b.Config.Output)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer reporter.Stop()
|
|
}
|
|
|
|
if b.Config.MetricLogging == nil || b.Config.MetricLogging.Enabled() {
|
|
reporter, err := log.MakeReporter(b.Info, b.Config.MetricLogging)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer reporter.Stop()
|
|
}
|
|
|
|
// If -configtest was specified, exit now prior to run.
|
|
if cfgfile.IsTestConfig() {
|
|
cfgwarn.Deprecate("6.0", "-configtest flag has been deprecated, use configtest subcommand")
|
|
fmt.Println("Config OK")
|
|
return beat.GracefulExit
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
svc.HandleSignals(beater.Stop, cancel)
|
|
|
|
err = b.loadDashboards(ctx, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if setup && b.SetupMLCallback != nil {
|
|
err = b.SetupMLCallback(&b.Beat, b.Config.Kibana)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
logp.Info("%s start running.", b.Info.Beat)
|
|
|
|
if b.Config.HTTP.Enabled() {
|
|
api.Start(b.Config.HTTP)
|
|
}
|
|
|
|
// Launch config manager
|
|
b.ConfigManager.Start()
|
|
defer b.ConfigManager.Stop()
|
|
|
|
return beater.Run(&b.Beat)
|
|
}
|
|
|
|
// TestConfig check all settings are ok and the beat can be run
|
|
func (b *Beat) TestConfig(bt beat.Creator) error {
|
|
return handleError(func() error {
|
|
err := b.Init()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Create beater to ensure all settings are OK
|
|
_, err = b.createBeater(bt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
fmt.Println("Config OK")
|
|
return beat.GracefulExit
|
|
}())
|
|
}
|
|
|
|
// Setup registers ES index template, kibana dashboards, ml jobs and pipelines.
|
|
func (b *Beat) Setup(bt beat.Creator, template, setupDashboards, machineLearning, pipelines bool) error {
|
|
return handleError(func() error {
|
|
err := b.Init()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Tell the beat that we're in the setup command
|
|
b.InSetupCmd = true
|
|
|
|
// Create beater to give it the opportunity to set loading callbacks
|
|
_, err = b.createBeater(bt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if template {
|
|
outCfg := b.Config.Output
|
|
|
|
if outCfg.Name() != "elasticsearch" {
|
|
return fmt.Errorf("Template loading requested but the Elasticsearch output is not configured/enabled")
|
|
}
|
|
|
|
esConfig := outCfg.Config()
|
|
if tmplCfg := b.Config.Template; tmplCfg == nil || tmplCfg.Enabled() {
|
|
loadCallback, err := b.templateLoadingCallback()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
esClient, err := elasticsearch.NewConnectedClient(esConfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Load template
|
|
err = loadCallback(esClient)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
fmt.Println("Loaded index template")
|
|
}
|
|
|
|
if setupDashboards {
|
|
fmt.Println("Loading dashboards (Kibana must be running and reachable)")
|
|
err = b.loadDashboards(context.Background(), true)
|
|
|
|
if err != nil {
|
|
switch err := errw.Cause(err).(type) {
|
|
case *dashboards.ErrNotFound:
|
|
fmt.Printf("Skipping loading dashboards, %+v\n", err)
|
|
default:
|
|
return err
|
|
}
|
|
} else {
|
|
fmt.Println("Loaded dashboards")
|
|
}
|
|
}
|
|
|
|
if machineLearning && b.SetupMLCallback != nil {
|
|
err = b.SetupMLCallback(&b.Beat, b.Config.Kibana)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Println("Loaded machine learning job configurations")
|
|
}
|
|
|
|
if pipelines && b.OverwritePipelinesCallback != nil {
|
|
esConfig := b.Config.Output.Config()
|
|
err = b.OverwritePipelinesCallback(esConfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
fmt.Println("Loaded Ingest pipelines")
|
|
}
|
|
|
|
return nil
|
|
}())
|
|
}
|
|
|
|
// handleFlags parses the command line flags. It handles the '-version' flag
|
|
// and invokes the HandleFlags callback if implemented by the Beat.
|
|
func (b *Beat) handleFlags() error {
|
|
flag.Parse()
|
|
|
|
if printVersion {
|
|
cfgwarn.Deprecate("6.0", "-version flag has been deprecated, use version subcommand")
|
|
fmt.Printf("%s version %s (%s), libbeat %s\n",
|
|
b.Info.Beat, b.Info.Version, runtime.GOARCH, version.GetDefaultVersion())
|
|
return beat.GracefulExit
|
|
}
|
|
|
|
return cfgfile.HandleFlags()
|
|
}
|
|
|
|
// config reads the configuration file from disk, parses the common options
|
|
// defined in BeatConfig, initializes logging, and set GOMAXPROCS if defined
|
|
// in the config. Lastly it invokes the Config method implemented by the beat.
|
|
func (b *Beat) configure(settings Settings) error {
|
|
var err error
|
|
|
|
cfg, err := cfgfile.Load("", settings.ConfigOverrides)
|
|
if err != nil {
|
|
return fmt.Errorf("error loading config file: %v", err)
|
|
}
|
|
|
|
// We have to initialize the keystore before any unpack or merging the cloud
|
|
// options.
|
|
keystoreCfg, _ := cfg.Child("keystore", -1)
|
|
defaultPathConfig, _ := cfg.String("path.config", -1)
|
|
defaultPathConfig = filepath.Join(defaultPathConfig, fmt.Sprintf("%s.keystore", b.Info.Beat))
|
|
store, err := keystore.Factory(keystoreCfg, defaultPathConfig)
|
|
if err != nil {
|
|
return fmt.Errorf("could not initialize the keystore: %v", err)
|
|
}
|
|
|
|
if settings.DisableConfigResolver {
|
|
common.OverwriteConfigOpts(obfuscateConfigOpts())
|
|
} else {
|
|
// TODO: Allow the options to be more flexible for dynamic changes
|
|
common.OverwriteConfigOpts(configOpts(store))
|
|
}
|
|
|
|
b.keystore = store
|
|
err = cloudid.OverwriteSettings(cfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
b.RawConfig = cfg
|
|
err = cfg.Unpack(&b.Config)
|
|
if err != nil {
|
|
return fmt.Errorf("error unpacking config data: %v", err)
|
|
}
|
|
|
|
b.Beat.Config = &b.Config.BeatConfig
|
|
|
|
err = cfgwarn.CheckRemoved5xSettings(cfg, "queue_size", "bulk_queue_size")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if name := b.Config.Name; name != "" {
|
|
b.Info.Name = name
|
|
}
|
|
|
|
err = paths.InitPaths(&b.Config.Path)
|
|
if err != nil {
|
|
return fmt.Errorf("error setting default paths: %v", err)
|
|
}
|
|
|
|
if err := configure.Logging(b.Info.Beat, b.Config.Logging); err != nil {
|
|
return fmt.Errorf("error initializing logging: %v", err)
|
|
}
|
|
|
|
// log paths values to help with troubleshooting
|
|
logp.Info(paths.Paths.String())
|
|
|
|
err = b.loadMeta()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
logp.Info("Beat UUID: %v", b.Info.UUID)
|
|
|
|
// initialize config manager
|
|
b.ConfigManager, err = management.Factory()(b.Config.Management, reload.Register, b.Beat.Info.UUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := b.ConfigManager.CheckRawConfig(b.RawConfig); err != nil {
|
|
return err
|
|
}
|
|
|
|
if maxProcs := b.Config.MaxProcs; maxProcs > 0 {
|
|
runtime.GOMAXPROCS(maxProcs)
|
|
}
|
|
|
|
b.Beat.BeatConfig, err = b.BeatConfig()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (b *Beat) loadMeta() error {
|
|
type meta struct {
|
|
UUID uuid.UUID `json:"uuid"`
|
|
}
|
|
|
|
metaPath := paths.Resolve(paths.Data, "meta.json")
|
|
logp.Debug("beat", "Beat metadata path: %v", metaPath)
|
|
|
|
f, err := openRegular(metaPath)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
return fmt.Errorf("Beat meta file failed to open: %s", err)
|
|
}
|
|
|
|
if err == nil {
|
|
m := meta{}
|
|
if err := json.NewDecoder(f).Decode(&m); err != nil {
|
|
f.Close()
|
|
return fmt.Errorf("Beat meta file reading error: %v", err)
|
|
}
|
|
|
|
f.Close()
|
|
valid := m.UUID != uuid.Nil
|
|
if valid {
|
|
b.Info.UUID = m.UUID
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// file does not exist or UUID is invalid, let's create a new one
|
|
|
|
// write temporary file first
|
|
tempFile := metaPath + ".new"
|
|
f, err = os.OpenFile(tempFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to create Beat meta file: %s", err)
|
|
}
|
|
|
|
err = json.NewEncoder(f).Encode(meta{UUID: b.Info.UUID})
|
|
f.Close()
|
|
if err != nil {
|
|
return fmt.Errorf("Beat meta file failed to write: %s", err)
|
|
}
|
|
|
|
// move temporary file into final location
|
|
err = file.SafeFileRotate(metaPath, tempFile)
|
|
return err
|
|
}
|
|
|
|
func openRegular(filename string) (*os.File, error) {
|
|
f, err := os.Open(filename)
|
|
if err != nil {
|
|
return f, err
|
|
}
|
|
|
|
info, err := f.Stat()
|
|
if err != nil {
|
|
f.Close()
|
|
return nil, err
|
|
}
|
|
|
|
if !info.Mode().IsRegular() {
|
|
f.Close()
|
|
if info.IsDir() {
|
|
return nil, fmt.Errorf("%s is a directory", filename)
|
|
}
|
|
return nil, fmt.Errorf("%s is not a regular file", filename)
|
|
}
|
|
|
|
return f, nil
|
|
}
|
|
|
|
func (b *Beat) loadDashboards(ctx context.Context, force bool) error {
|
|
if setup || force {
|
|
// -setup implies dashboards.enabled=true
|
|
if b.Config.Dashboards == nil {
|
|
b.Config.Dashboards = common.NewConfig()
|
|
}
|
|
err := b.Config.Dashboards.SetBool("enabled", -1, true)
|
|
if err != nil {
|
|
return fmt.Errorf("Error setting dashboard.enabled=true: %v", err)
|
|
}
|
|
}
|
|
|
|
if b.Config.Dashboards.Enabled() {
|
|
var esConfig *common.Config
|
|
|
|
if b.Config.Output.Name() == "elasticsearch" {
|
|
esConfig = b.Config.Output.Config()
|
|
}
|
|
err := dashboards.ImportDashboards(ctx, b.Info.Beat, b.Info.Hostname, paths.Resolve(paths.Home, ""),
|
|
b.Config.Kibana, esConfig, b.Config.Dashboards, nil)
|
|
if err != nil {
|
|
return errw.Wrap(err, "Error importing Kibana dashboards")
|
|
}
|
|
logp.Info("Kibana dashboards successfully loaded.")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// registerTemplateLoading registers the loading of the template as a callback with
|
|
// the elasticsearch output. It is important the the registration happens before
|
|
// the publisher is created.
|
|
func (b *Beat) registerTemplateLoading() error {
|
|
var cfg template.TemplateConfig
|
|
|
|
// Check if outputting to file is enabled, and output to file if it is
|
|
if b.Config.Template.Enabled() {
|
|
err := b.Config.Template.Unpack(&cfg)
|
|
if err != nil {
|
|
return fmt.Errorf("unpacking template config fails: %v", err)
|
|
}
|
|
}
|
|
|
|
// Loads template by default if esOutput is enabled
|
|
if b.Config.Output.Name() == "elasticsearch" {
|
|
|
|
// Get ES Index name for comparison
|
|
esCfg := struct {
|
|
Index string `config:"index"`
|
|
}{}
|
|
err := b.Config.Output.Config().Unpack(&esCfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if esCfg.Index != "" && (cfg.Name == "" || cfg.Pattern == "") && (b.Config.Template == nil || b.Config.Template.Enabled()) {
|
|
return fmt.Errorf("setup.template.name and setup.template.pattern have to be set if index name is modified.")
|
|
}
|
|
|
|
if b.Config.Template == nil || (b.Config.Template != nil && b.Config.Template.Enabled()) {
|
|
|
|
// load template through callback to make sure it is also loaded
|
|
// on reconnecting
|
|
callback, err := b.templateLoadingCallback()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
elasticsearch.RegisterConnectCallback(callback)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Build and return a callback to load index template into ES
|
|
func (b *Beat) templateLoadingCallback() (func(esClient *elasticsearch.Client) error, error) {
|
|
callback := func(esClient *elasticsearch.Client) error {
|
|
if b.Config.Template == nil {
|
|
b.Config.Template = common.NewConfig()
|
|
}
|
|
|
|
loader, err := template.NewLoader(b.Config.Template, esClient, b.Info, b.Fields)
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating Elasticsearch template loader: %v", err)
|
|
}
|
|
|
|
err = loader.Load()
|
|
if err != nil {
|
|
return fmt.Errorf("Error loading Elasticsearch template: %v", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
return callback, nil
|
|
}
|
|
|
|
// handleError handles the given error by logging it and then returning the
|
|
// error. If the err is nil or is a GracefulExit error then the method will
|
|
// return nil without logging anything.
|
|
func handleError(err error) error {
|
|
if err == nil || err == beat.GracefulExit {
|
|
return nil
|
|
}
|
|
|
|
// logp may not be initialized so log the err to stderr too.
|
|
logp.Critical("Exiting: %v", err)
|
|
fmt.Fprintf(os.Stderr, "Exiting: %v\n", err)
|
|
return err
|
|
}
|
|
|
|
// logSystemInfo logs information about this system for situational awareness
|
|
// in debugging. This information includes data about the beat, build, go
|
|
// runtime, host, and process. If any of the data is not available it will be
|
|
// omitted.
|
|
func logSystemInfo(info beat.Info) {
|
|
defer logp.Recover("An unexpected error occurred while collecting " +
|
|
"information about the system.")
|
|
log := logp.NewLogger("beat").With(logp.Namespace("system_info"))
|
|
|
|
// Beat
|
|
beat := common.MapStr{
|
|
"type": info.Beat,
|
|
"uuid": info.UUID,
|
|
"path": common.MapStr{
|
|
"config": paths.Resolve(paths.Config, ""),
|
|
"data": paths.Resolve(paths.Data, ""),
|
|
"home": paths.Resolve(paths.Home, ""),
|
|
"logs": paths.Resolve(paths.Logs, ""),
|
|
},
|
|
}
|
|
log.Infow("Beat info", "beat", beat)
|
|
|
|
// Build
|
|
build := common.MapStr{
|
|
"commit": version.Commit(),
|
|
"time": version.BuildTime(),
|
|
"version": info.Version,
|
|
"libbeat": version.GetDefaultVersion(),
|
|
}
|
|
log.Infow("Build info", "build", build)
|
|
|
|
// Go Runtime
|
|
log.Infow("Go runtime info", "go", sysinfo.Go())
|
|
|
|
// Host
|
|
if host, err := sysinfo.Host(); err == nil {
|
|
log.Infow("Host info", "host", host.Info())
|
|
}
|
|
|
|
// Process
|
|
if self, err := sysinfo.Self(); err == nil {
|
|
process := common.MapStr{}
|
|
|
|
if info, err := self.Info(); err == nil {
|
|
process["name"] = info.Name
|
|
process["pid"] = info.PID
|
|
process["ppid"] = info.PPID
|
|
process["cwd"] = info.CWD
|
|
process["exe"] = info.Exe
|
|
process["start_time"] = info.StartTime
|
|
}
|
|
|
|
if proc, ok := self.(types.Seccomp); ok {
|
|
if seccomp, err := proc.Seccomp(); err == nil {
|
|
process["seccomp"] = seccomp
|
|
}
|
|
}
|
|
|
|
if proc, ok := self.(types.Capabilities); ok {
|
|
if caps, err := proc.Capabilities(); err == nil {
|
|
process["capabilities"] = caps
|
|
}
|
|
}
|
|
|
|
if len(process) > 0 {
|
|
log.Infow("Process info", "process", process)
|
|
}
|
|
}
|
|
}
|
|
|
|
// configOpts returns ucfg config options with a resolver linked to the current keystore.
|
|
// TODO: Refactor to allow insert into the config option array without having to redefine everything
|
|
func configOpts(store keystore.Keystore) []ucfg.Option {
|
|
return []ucfg.Option{
|
|
ucfg.PathSep("."),
|
|
ucfg.Resolve(keystore.ResolverWrap(store)),
|
|
ucfg.ResolveEnv,
|
|
ucfg.VarExp,
|
|
}
|
|
}
|
|
|
|
// obfuscateConfigOpts disables any resolvers in the configuration, instead we return the field
|
|
// reference string directly.
|
|
func obfuscateConfigOpts() []ucfg.Option {
|
|
return []ucfg.Option{
|
|
ucfg.PathSep("."),
|
|
ucfg.ResolveNOOP,
|
|
}
|
|
}
|