448 lines
13 KiB
Go
448 lines
13 KiB
Go
// Licensed to Elasticsearch B.V. under one or more contributor
|
|
// license agreements. See the NOTICE file distributed with
|
|
// this work for additional information regarding copyright
|
|
// ownership. Elasticsearch B.V. licenses this file to you under
|
|
// the Apache License, Version 2.0 (the "License"); you may
|
|
// not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
// +build !integration
|
|
|
|
package fileset
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"github.com/elastic/beats/libbeat/common"
|
|
"github.com/elastic/beats/libbeat/paths"
|
|
)
|
|
|
|
func load(t *testing.T, from interface{}) *common.Config {
|
|
config, err := common.NewConfigFrom(from)
|
|
if err != nil {
|
|
t.Fatalf("Config err: %v", err)
|
|
}
|
|
return config
|
|
}
|
|
|
|
func TestNewModuleRegistry(t *testing.T) {
|
|
modulesPath, err := filepath.Abs("../module")
|
|
assert.NoError(t, err)
|
|
|
|
configs := []*ModuleConfig{
|
|
&ModuleConfig{Module: "nginx"},
|
|
&ModuleConfig{Module: "mysql"},
|
|
&ModuleConfig{Module: "system"},
|
|
&ModuleConfig{Module: "auditd"},
|
|
}
|
|
|
|
reg, err := newModuleRegistry(modulesPath, configs, nil, "5.2.0")
|
|
assert.NoError(t, err)
|
|
assert.NotNil(t, reg)
|
|
|
|
expectedModules := map[string][]string{
|
|
"auditd": {"log"},
|
|
"nginx": {"access", "error"},
|
|
"mysql": {"slowlog", "error"},
|
|
"system": {"syslog", "auth"},
|
|
}
|
|
|
|
assert.Equal(t, len(expectedModules), len(reg.registry))
|
|
for name, filesets := range reg.registry {
|
|
expectedFilesets, exists := expectedModules[name]
|
|
assert.True(t, exists)
|
|
|
|
assert.Equal(t, len(expectedFilesets), len(filesets))
|
|
for _, fileset := range expectedFilesets {
|
|
fs := filesets[fileset]
|
|
assert.NotNil(t, fs)
|
|
}
|
|
}
|
|
|
|
for module, filesets := range reg.registry {
|
|
for name, fileset := range filesets {
|
|
cfg, err := fileset.getInputConfig()
|
|
assert.NoError(t, err, fmt.Sprintf("module: %s, fileset: %s", module, name))
|
|
|
|
moduleName, err := cfg.String("_module_name", -1)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, module, moduleName)
|
|
|
|
filesetName, err := cfg.String("_fileset_name", -1)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, name, filesetName)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNewModuleRegistryConfig(t *testing.T) {
|
|
modulesPath, err := filepath.Abs("../module")
|
|
assert.NoError(t, err)
|
|
|
|
falseVar := false
|
|
|
|
configs := []*ModuleConfig{
|
|
&ModuleConfig{
|
|
Module: "nginx",
|
|
Filesets: map[string]*FilesetConfig{
|
|
"access": {
|
|
Var: map[string]interface{}{
|
|
"paths": []interface{}{"/hello/test"},
|
|
},
|
|
},
|
|
"error": {
|
|
Enabled: &falseVar,
|
|
},
|
|
},
|
|
},
|
|
&ModuleConfig{
|
|
Module: "mysql",
|
|
Enabled: &falseVar,
|
|
},
|
|
}
|
|
|
|
reg, err := newModuleRegistry(modulesPath, configs, nil, "5.2.0")
|
|
assert.NoError(t, err)
|
|
assert.NotNil(t, reg)
|
|
|
|
nginxAccess := reg.registry["nginx"]["access"]
|
|
assert.NotNil(t, nginxAccess)
|
|
assert.Equal(t, []interface{}{"/hello/test"}, nginxAccess.vars["paths"])
|
|
|
|
assert.NotContains(t, reg.registry["nginx"], "error")
|
|
}
|
|
|
|
func TestApplyOverrides(t *testing.T) {
|
|
falseVar := false
|
|
trueVar := true
|
|
|
|
tests := []struct {
|
|
name string
|
|
fcfg FilesetConfig
|
|
module, fileset string
|
|
overrides *ModuleOverrides
|
|
expected FilesetConfig
|
|
}{
|
|
{
|
|
name: "var overrides",
|
|
fcfg: FilesetConfig{
|
|
Var: map[string]interface{}{
|
|
"a": "test",
|
|
"b.c": "test",
|
|
},
|
|
},
|
|
module: "nginx",
|
|
fileset: "access",
|
|
overrides: &ModuleOverrides{
|
|
"nginx": map[string]*common.Config{
|
|
"access": load(t, map[string]interface{}{
|
|
"var.a": "test1",
|
|
"var.b.c": "test2"}),
|
|
},
|
|
},
|
|
expected: FilesetConfig{
|
|
Var: map[string]interface{}{
|
|
"a": "test1",
|
|
"b": map[string]interface{}{"c": "test2"},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "enable and var overrides",
|
|
fcfg: FilesetConfig{
|
|
Enabled: &falseVar,
|
|
Var: map[string]interface{}{
|
|
"paths": []string{"/var/log/nginx"},
|
|
},
|
|
},
|
|
module: "nginx",
|
|
fileset: "access",
|
|
overrides: &ModuleOverrides{
|
|
"nginx": map[string]*common.Config{
|
|
"access": load(t, map[string]interface{}{
|
|
"enabled": true,
|
|
"var.paths": []interface{}{"/var/local/nginx/log"}}),
|
|
},
|
|
},
|
|
expected: FilesetConfig{
|
|
Enabled: &trueVar,
|
|
Var: map[string]interface{}{
|
|
"paths": []interface{}{"/var/local/nginx/log"},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "prospector overrides",
|
|
fcfg: FilesetConfig{},
|
|
module: "nginx",
|
|
fileset: "access",
|
|
overrides: &ModuleOverrides{
|
|
"nginx": map[string]*common.Config{
|
|
"access": load(t, map[string]interface{}{
|
|
"prospector.close_eof": true,
|
|
}),
|
|
},
|
|
},
|
|
expected: FilesetConfig{
|
|
Input: map[string]interface{}{
|
|
"close_eof": true,
|
|
},
|
|
Prospector: map[string]interface{}{
|
|
"close_eof": true,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "input overrides",
|
|
fcfg: FilesetConfig{},
|
|
module: "nginx",
|
|
fileset: "access",
|
|
overrides: &ModuleOverrides{
|
|
"nginx": map[string]*common.Config{
|
|
"access": load(t, map[string]interface{}{
|
|
"input.close_eof": true,
|
|
}),
|
|
},
|
|
},
|
|
expected: FilesetConfig{
|
|
Input: map[string]interface{}{
|
|
"close_eof": true,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
result, err := applyOverrides(&test.fcfg, test.module, test.fileset, test.overrides)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, &test.expected, result, test.name)
|
|
}
|
|
}
|
|
|
|
func TestAppendWithoutDuplicates(t *testing.T) {
|
|
falseVar := false
|
|
tests := []struct {
|
|
name string
|
|
configs []*ModuleConfig
|
|
modules []string
|
|
expected []*ModuleConfig
|
|
}{
|
|
{
|
|
name: "just modules",
|
|
configs: []*ModuleConfig{},
|
|
modules: []string{"moduleA", "moduleB", "moduleC"},
|
|
expected: []*ModuleConfig{
|
|
&ModuleConfig{Module: "moduleA"},
|
|
&ModuleConfig{Module: "moduleB"},
|
|
&ModuleConfig{Module: "moduleC"},
|
|
},
|
|
},
|
|
{
|
|
name: "eliminate a duplicate, no override",
|
|
configs: []*ModuleConfig{
|
|
&ModuleConfig{
|
|
Module: "moduleB",
|
|
Filesets: map[string]*FilesetConfig{
|
|
"fileset": {
|
|
Var: map[string]interface{}{
|
|
"paths": "test",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
modules: []string{"moduleA", "moduleB", "moduleC"},
|
|
expected: []*ModuleConfig{
|
|
&ModuleConfig{
|
|
Module: "moduleB",
|
|
Filesets: map[string]*FilesetConfig{
|
|
"fileset": {
|
|
Var: map[string]interface{}{
|
|
"paths": "test",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
&ModuleConfig{Module: "moduleA"},
|
|
&ModuleConfig{Module: "moduleC"},
|
|
},
|
|
},
|
|
{
|
|
name: "disabled config",
|
|
configs: []*ModuleConfig{
|
|
&ModuleConfig{
|
|
Module: "moduleB",
|
|
Enabled: &falseVar,
|
|
Filesets: map[string]*FilesetConfig{
|
|
"fileset": {
|
|
Var: map[string]interface{}{
|
|
"paths": "test",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
modules: []string{"moduleA", "moduleB", "moduleC"},
|
|
expected: []*ModuleConfig{
|
|
&ModuleConfig{
|
|
Module: "moduleB",
|
|
Enabled: &falseVar,
|
|
Filesets: map[string]*FilesetConfig{
|
|
"fileset": {
|
|
Var: map[string]interface{}{
|
|
"paths": "test",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
&ModuleConfig{Module: "moduleA"},
|
|
&ModuleConfig{Module: "moduleB"},
|
|
&ModuleConfig{Module: "moduleC"},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
result, err := appendWithoutDuplicates(test.configs, test.modules)
|
|
assert.NoError(t, err, test.name)
|
|
assert.Equal(t, test.expected, result, test.name)
|
|
}
|
|
}
|
|
|
|
func TestMcfgFromConfig(t *testing.T) {
|
|
falseVar := false
|
|
tests := []struct {
|
|
name string
|
|
config *common.Config
|
|
expected ModuleConfig
|
|
}{
|
|
{
|
|
name: "disable fileset",
|
|
config: load(t, map[string]interface{}{
|
|
"module": "nginx",
|
|
"error.enabled": false,
|
|
}),
|
|
expected: ModuleConfig{
|
|
Module: "nginx",
|
|
Filesets: map[string]*FilesetConfig{
|
|
"error": {
|
|
Enabled: &falseVar,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "set variable",
|
|
config: load(t, map[string]interface{}{
|
|
"module": "nginx",
|
|
"access.var.test": false,
|
|
}),
|
|
expected: ModuleConfig{
|
|
Module: "nginx",
|
|
Filesets: map[string]*FilesetConfig{
|
|
"access": {
|
|
Var: map[string]interface{}{
|
|
"test": false,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
result, err := mcfgFromConfig(test.config)
|
|
assert.NoError(t, err, test.name)
|
|
assert.Equal(t, test.expected.Module, result.Module)
|
|
assert.Equal(t, len(test.expected.Filesets), len(result.Filesets))
|
|
for name, fileset := range test.expected.Filesets {
|
|
assert.Equal(t, fileset, result.Filesets[name])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMissingModuleFolder(t *testing.T) {
|
|
home := paths.Paths.Home
|
|
paths.Paths.Home = "/no/such/path"
|
|
defer func() { paths.Paths.Home = home }()
|
|
|
|
configs := []*common.Config{
|
|
load(t, map[string]interface{}{"module": "nginx"}),
|
|
}
|
|
|
|
reg, err := NewModuleRegistry(configs, "5.2.0", true)
|
|
assert.NoError(t, err)
|
|
assert.NotNil(t, reg)
|
|
|
|
// this should return an empty list, but no error
|
|
inputs, err := reg.GetInputConfigs()
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 0, len(inputs))
|
|
}
|
|
|
|
func TestInterpretError(t *testing.T) {
|
|
tests := []struct {
|
|
Test string
|
|
Input string
|
|
Output string
|
|
}{
|
|
{
|
|
Test: "geoip not installed",
|
|
Input: `{"error":{"root_cause":[{"type":"parse_exception","reason":"No processor type exists with name [geoip]","header":{"processor_type":"geoip"}}],"type":"parse_exception","reason":"No processor type exists with name [geoip]","header":{"processor_type":"geoip"}},"status":400}`,
|
|
Output: "This module requires the ingest-geoip plugin to be installed in Elasticsearch. You can install it using the following command in the Elasticsearch home directory:\n sudo bin/elasticsearch-plugin install ingest-geoip",
|
|
},
|
|
{
|
|
Test: "user-agent not installed",
|
|
Input: `{"error":{"root_cause":[{"type":"parse_exception","reason":"No processor type exists with name [user_agent]","header":{"processor_type":"user_agent"}}],"type":"parse_exception","reason":"No processor type exists with name [user_agent]","header":{"processor_type":"user_agent"}},"status":400}`,
|
|
Output: "This module requires the ingest-user-agent plugin to be installed in Elasticsearch. You can install it using the following command in the Elasticsearch home directory:\n sudo bin/elasticsearch-plugin install ingest-user-agent",
|
|
},
|
|
{
|
|
Test: "other plugin not installed",
|
|
Input: `{"error":{"root_cause":[{"type":"parse_exception","reason":"No processor type exists with name [hello_test]","header":{"processor_type":"hello_test"}}],"type":"parse_exception","reason":"No processor type exists with name [hello_test]","header":{"processor_type":"hello_test"}},"status":400}`,
|
|
Output: "This module requires an Elasticsearch plugin that provides the hello_test processor. " +
|
|
"Please visit the Elasticsearch documentation for instructions on how to install this plugin. " +
|
|
"Response body: " + `{"error":{"root_cause":[{"type":"parse_exception","reason":"No processor type exists with name [hello_test]","header":{"processor_type":"hello_test"}}],"type":"parse_exception","reason":"No processor type exists with name [hello_test]","header":{"processor_type":"hello_test"}},"status":400}`,
|
|
},
|
|
{
|
|
Test: "Elasticsearch 2.4",
|
|
Input: `{"error":{"root_cause":[{"type":"invalid_index_name_exception","reason":"Invalid index name [_ingest], must not start with '_'","index":"_ingest"}],"type":"invalid_index_name_exception","reason":"Invalid index name [_ingest], must not start with '_'","index":"_ingest"},"status":400}`,
|
|
Output: `The Ingest Node functionality seems to be missing from Elasticsearch. The Filebeat modules require Elasticsearch >= 5.0. This is the response I got from Elasticsearch: {"error":{"root_cause":[{"type":"invalid_index_name_exception","reason":"Invalid index name [_ingest], must not start with '_'","index":"_ingest"}],"type":"invalid_index_name_exception","reason":"Invalid index name [_ingest], must not start with '_'","index":"_ingest"},"status":400}`,
|
|
},
|
|
{
|
|
Test: "Elasticsearch 1.7",
|
|
Input: `{"error":"InvalidIndexNameException[[_ingest] Invalid index name [_ingest], must not start with '_']","status":400}`,
|
|
Output: `The Filebeat modules require Elasticsearch >= 5.0. This is the response I got from Elasticsearch: {"error":"InvalidIndexNameException[[_ingest] Invalid index name [_ingest], must not start with '_']","status":400}`,
|
|
},
|
|
{
|
|
Test: "bad json",
|
|
Input: `blah`,
|
|
Output: `couldn't load pipeline: test. Additionally, error decoding response body: blah`,
|
|
},
|
|
{
|
|
Test: "another error",
|
|
Input: `{"error":{"root_cause":[{"type":"test","reason":""}],"type":"test","reason":""},"status":400}`,
|
|
Output: "couldn't load pipeline: test. Response body: " +
|
|
`{"error":{"root_cause":[{"type":"test","reason":""}],"type":"test","reason":""},"status":400}`,
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
errResult := interpretError(errors.New("test"), []byte(test.Input))
|
|
assert.Equal(t, errResult.Error(), test.Output, test.Test)
|
|
}
|
|
}
|