youtubebeat/vendor/github.com/elastic/beats/filebeat/fileset/modules_test.go

447 lines
13 KiB
Go

// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// +build !integration
package fileset
import (
"errors"
"fmt"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/paths"
)
func load(t *testing.T, from interface{}) *common.Config {
config, err := common.NewConfigFrom(from)
if err != nil {
t.Fatalf("Config err: %v", err)
}
return config
}
func TestNewModuleRegistry(t *testing.T) {
modulesPath, err := filepath.Abs("../module")
assert.NoError(t, err)
configs := []*ModuleConfig{
&ModuleConfig{Module: "nginx"},
&ModuleConfig{Module: "mysql"},
&ModuleConfig{Module: "system"},
&ModuleConfig{Module: "auditd"},
}
reg, err := newModuleRegistry(modulesPath, configs, nil, "5.2.0")
assert.NoError(t, err)
assert.NotNil(t, reg)
expectedModules := map[string][]string{
"auditd": {"log"},
"nginx": {"access", "error"},
"mysql": {"slowlog", "error"},
"system": {"syslog", "auth"},
}
assert.Equal(t, len(expectedModules), len(reg.registry))
for name, filesets := range reg.registry {
expectedFilesets, exists := expectedModules[name]
assert.True(t, exists)
assert.Equal(t, len(expectedFilesets), len(filesets))
for _, fileset := range expectedFilesets {
fs := filesets[fileset]
assert.NotNil(t, fs)
}
}
for module, filesets := range reg.registry {
for name, fileset := range filesets {
cfg, err := fileset.getInputConfig()
assert.NoError(t, err, fmt.Sprintf("module: %s, fileset: %s", module, name))
moduleName, err := cfg.String("_module_name", -1)
assert.NoError(t, err)
assert.Equal(t, module, moduleName)
filesetName, err := cfg.String("_fileset_name", -1)
assert.NoError(t, err)
assert.Equal(t, name, filesetName)
}
}
}
func TestNewModuleRegistryConfig(t *testing.T) {
modulesPath, err := filepath.Abs("../module")
assert.NoError(t, err)
falseVar := false
configs := []*ModuleConfig{
&ModuleConfig{
Module: "nginx",
Filesets: map[string]*FilesetConfig{
"access": {
Var: map[string]interface{}{
"paths": []interface{}{"/hello/test"},
},
},
"error": {
Enabled: &falseVar,
},
},
},
&ModuleConfig{
Module: "mysql",
Enabled: &falseVar,
},
}
reg, err := newModuleRegistry(modulesPath, configs, nil, "5.2.0")
assert.NoError(t, err)
assert.NotNil(t, reg)
nginxAccess := reg.registry["nginx"]["access"]
assert.NotNil(t, nginxAccess)
assert.Equal(t, []interface{}{"/hello/test"}, nginxAccess.vars["paths"])
assert.NotContains(t, reg.registry["nginx"], "error")
}
func TestApplyOverrides(t *testing.T) {
falseVar := false
trueVar := true
tests := []struct {
name string
fcfg FilesetConfig
module, fileset string
overrides *ModuleOverrides
expected FilesetConfig
}{
{
name: "var overrides",
fcfg: FilesetConfig{
Var: map[string]interface{}{
"a": "test",
"b.c": "test",
},
},
module: "nginx",
fileset: "access",
overrides: &ModuleOverrides{
"nginx": map[string]*common.Config{
"access": load(t, map[string]interface{}{
"var.a": "test1",
"var.b.c": "test2"}),
},
},
expected: FilesetConfig{
Var: map[string]interface{}{
"a": "test1",
"b": map[string]interface{}{"c": "test2"},
},
},
},
{
name: "enable and var overrides",
fcfg: FilesetConfig{
Enabled: &falseVar,
Var: map[string]interface{}{
"paths": []string{"/var/log/nginx"},
},
},
module: "nginx",
fileset: "access",
overrides: &ModuleOverrides{
"nginx": map[string]*common.Config{
"access": load(t, map[string]interface{}{
"enabled": true,
"var.paths": []interface{}{"/var/local/nginx/log"}}),
},
},
expected: FilesetConfig{
Enabled: &trueVar,
Var: map[string]interface{}{
"paths": []interface{}{"/var/local/nginx/log"},
},
},
},
{
name: "prospector overrides",
fcfg: FilesetConfig{},
module: "nginx",
fileset: "access",
overrides: &ModuleOverrides{
"nginx": map[string]*common.Config{
"access": load(t, map[string]interface{}{
"prospector.close_eof": true,
}),
},
},
expected: FilesetConfig{
Input: map[string]interface{}{
"close_eof": true,
},
Prospector: map[string]interface{}{
"close_eof": true,
},
},
},
{
name: "input overrides",
fcfg: FilesetConfig{},
module: "nginx",
fileset: "access",
overrides: &ModuleOverrides{
"nginx": map[string]*common.Config{
"access": load(t, map[string]interface{}{
"input.close_eof": true,
}),
},
},
expected: FilesetConfig{
Input: map[string]interface{}{
"close_eof": true,
},
},
},
}
for _, test := range tests {
result, err := applyOverrides(&test.fcfg, test.module, test.fileset, test.overrides)
assert.NoError(t, err)
assert.Equal(t, &test.expected, result, test.name)
}
}
func TestAppendWithoutDuplicates(t *testing.T) {
falseVar := false
tests := []struct {
name string
configs []*ModuleConfig
modules []string
expected []*ModuleConfig
}{
{
name: "just modules",
configs: []*ModuleConfig{},
modules: []string{"moduleA", "moduleB", "moduleC"},
expected: []*ModuleConfig{
&ModuleConfig{Module: "moduleA"},
&ModuleConfig{Module: "moduleB"},
&ModuleConfig{Module: "moduleC"},
},
},
{
name: "eliminate a duplicate, no override",
configs: []*ModuleConfig{
&ModuleConfig{
Module: "moduleB",
Filesets: map[string]*FilesetConfig{
"fileset": {
Var: map[string]interface{}{
"paths": "test",
},
},
},
},
},
modules: []string{"moduleA", "moduleB", "moduleC"},
expected: []*ModuleConfig{
&ModuleConfig{
Module: "moduleB",
Filesets: map[string]*FilesetConfig{
"fileset": {
Var: map[string]interface{}{
"paths": "test",
},
},
},
},
&ModuleConfig{Module: "moduleA"},
&ModuleConfig{Module: "moduleC"},
},
},
{
name: "disabled config",
configs: []*ModuleConfig{
&ModuleConfig{
Module: "moduleB",
Enabled: &falseVar,
Filesets: map[string]*FilesetConfig{
"fileset": {
Var: map[string]interface{}{
"paths": "test",
},
},
},
},
},
modules: []string{"moduleA", "moduleB", "moduleC"},
expected: []*ModuleConfig{
&ModuleConfig{
Module: "moduleB",
Enabled: &falseVar,
Filesets: map[string]*FilesetConfig{
"fileset": {
Var: map[string]interface{}{
"paths": "test",
},
},
},
},
&ModuleConfig{Module: "moduleA"},
&ModuleConfig{Module: "moduleB"},
&ModuleConfig{Module: "moduleC"},
},
},
}
for _, test := range tests {
result, err := appendWithoutDuplicates(test.configs, test.modules)
assert.NoError(t, err, test.name)
assert.Equal(t, test.expected, result, test.name)
}
}
func TestMcfgFromConfig(t *testing.T) {
falseVar := false
tests := []struct {
name string
config *common.Config
expected ModuleConfig
}{
{
name: "disable fileset",
config: load(t, map[string]interface{}{
"module": "nginx",
"error.enabled": false,
}),
expected: ModuleConfig{
Module: "nginx",
Filesets: map[string]*FilesetConfig{
"error": {
Enabled: &falseVar,
},
},
},
},
{
name: "set variable",
config: load(t, map[string]interface{}{
"module": "nginx",
"access.var.test": false,
}),
expected: ModuleConfig{
Module: "nginx",
Filesets: map[string]*FilesetConfig{
"access": {
Var: map[string]interface{}{
"test": false,
},
},
},
},
},
}
for _, test := range tests {
result, err := mcfgFromConfig(test.config)
assert.NoError(t, err, test.name)
assert.Equal(t, test.expected.Module, result.Module)
assert.Equal(t, len(test.expected.Filesets), len(result.Filesets))
for name, fileset := range test.expected.Filesets {
assert.Equal(t, fileset, result.Filesets[name])
}
}
}
func TestMissingModuleFolder(t *testing.T) {
home := paths.Paths.Home
paths.Paths.Home = "/no/such/path"
defer func() { paths.Paths.Home = home }()
configs := []*common.Config{
load(t, map[string]interface{}{"module": "nginx"}),
}
reg, err := NewModuleRegistry(configs, "5.2.0", true)
assert.NoError(t, err)
assert.NotNil(t, reg)
// this should return an empty list, but no error
inputs, err := reg.GetInputConfigs()
assert.NoError(t, err)
assert.Equal(t, 0, len(inputs))
}
func TestInterpretError(t *testing.T) {
tests := []struct {
Test string
Input string
Output string
}{
{
Test: "geoip not installed",
Input: `{"error":{"root_cause":[{"type":"parse_exception","reason":"No processor type exists with name [geoip]","header":{"processor_type":"geoip"}}],"type":"parse_exception","reason":"No processor type exists with name [geoip]","header":{"processor_type":"geoip"}},"status":400}`,
Output: "This module requires the ingest-geoip plugin to be installed in Elasticsearch. You can install it using the following command in the Elasticsearch home directory:\n sudo bin/elasticsearch-plugin install ingest-geoip",
},
{
Test: "user-agent not installed",
Input: `{"error":{"root_cause":[{"type":"parse_exception","reason":"No processor type exists with name [user_agent]","header":{"processor_type":"user_agent"}}],"type":"parse_exception","reason":"No processor type exists with name [user_agent]","header":{"processor_type":"user_agent"}},"status":400}`,
Output: "This module requires the ingest-user-agent plugin to be installed in Elasticsearch. You can install it using the following command in the Elasticsearch home directory:\n sudo bin/elasticsearch-plugin install ingest-user-agent",
},
{
Test: "other plugin not installed",
Input: `{"error":{"root_cause":[{"type":"parse_exception","reason":"No processor type exists with name [hello_test]","header":{"processor_type":"hello_test"}}],"type":"parse_exception","reason":"No processor type exists with name [hello_test]","header":{"processor_type":"hello_test"}},"status":400}`,
Output: "This module requires an Elasticsearch plugin that provides the hello_test processor. " +
"Please visit the Elasticsearch documentation for instructions on how to install this plugin. " +
"Response body: " + `{"error":{"root_cause":[{"type":"parse_exception","reason":"No processor type exists with name [hello_test]","header":{"processor_type":"hello_test"}}],"type":"parse_exception","reason":"No processor type exists with name [hello_test]","header":{"processor_type":"hello_test"}},"status":400}`,
},
{
Test: "Elasticsearch 2.4",
Input: `{"error":{"root_cause":[{"type":"invalid_index_name_exception","reason":"Invalid index name [_ingest], must not start with '_'","index":"_ingest"}],"type":"invalid_index_name_exception","reason":"Invalid index name [_ingest], must not start with '_'","index":"_ingest"},"status":400}`,
Output: `The Ingest Node functionality seems to be missing from Elasticsearch. The Filebeat modules require Elasticsearch >= 5.0. This is the response I got from Elasticsearch: {"error":{"root_cause":[{"type":"invalid_index_name_exception","reason":"Invalid index name [_ingest], must not start with '_'","index":"_ingest"}],"type":"invalid_index_name_exception","reason":"Invalid index name [_ingest], must not start with '_'","index":"_ingest"},"status":400}`,
},
{
Test: "Elasticsearch 1.7",
Input: `{"error":"InvalidIndexNameException[[_ingest] Invalid index name [_ingest], must not start with '_']","status":400}`,
Output: `The Filebeat modules require Elasticsearch >= 5.0. This is the response I got from Elasticsearch: {"error":"InvalidIndexNameException[[_ingest] Invalid index name [_ingest], must not start with '_']","status":400}`,
},
{
Test: "bad json",
Input: `blah`,
Output: `couldn't load pipeline: test. Additionally, error decoding response body: blah`,
},
{
Test: "another error",
Input: `{"error":{"root_cause":[{"type":"test","reason":""}],"type":"test","reason":""},"status":400}`,
Output: "couldn't load pipeline: test. Response body: " +
`{"error":{"root_cause":[{"type":"test","reason":""}],"type":"test","reason":""},"status":400}`,
},
}
for _, test := range tests {
errResult := interpretError(errors.New("test"), []byte(test.Input))
assert.Equal(t, errResult.Error(), test.Output, test.Test)
}
}