youtubebeat/vendor/github.com/elastic/beats/libbeat/template/template.go

296 lines
7.1 KiB
Go

// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package template
import (
"fmt"
"sync"
"time"
"github.com/elastic/beats/libbeat/beat"
"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/common/cfgwarn"
"github.com/elastic/beats/libbeat/common/fmtstr"
"github.com/elastic/go-ucfg/yaml"
)
var (
// Defaults used in the template
defaultDateDetection = false
defaultTotalFieldsLimit = 10000
defaultNumberOfRoutingShards = 30
// Array to store dynamicTemplate parts in
dynamicTemplates []common.MapStr
defaultFields []string
)
type Template struct {
sync.Mutex
name string
pattern string
beatVersion common.Version
esVersion common.Version
config TemplateConfig
}
// New creates a new template instance
func New(beatVersion string, beatName string, esVersion string, config TemplateConfig) (*Template, error) {
bV, err := common.NewVersion(beatVersion)
if err != nil {
return nil, err
}
name := config.Name
if name == "" {
name = fmt.Sprintf("%s-%s", beatName, bV.String())
}
pattern := config.Pattern
if pattern == "" {
pattern = name + "-*"
}
event := &beat.Event{
Fields: common.MapStr{
"beat": common.MapStr{
"name": beatName,
"version": bV.String(),
},
},
Timestamp: time.Now(),
}
nameFormatter, err := fmtstr.CompileEvent(name)
if err != nil {
return nil, err
}
name, err = nameFormatter.Run(event)
if err != nil {
return nil, err
}
patternFormatter, err := fmtstr.CompileEvent(pattern)
if err != nil {
return nil, err
}
pattern, err = patternFormatter.Run(event)
if err != nil {
return nil, err
}
// In case no esVersion is set, it is assumed the same as beat version
if esVersion == "" {
esVersion = beatVersion
}
esV, err := common.NewVersion(esVersion)
if err != nil {
return nil, err
}
return &Template{
pattern: pattern,
name: name,
beatVersion: *bV,
esVersion: *esV,
config: config,
}, nil
}
func (t *Template) load(fields common.Fields) (common.MapStr, error) {
// Locking to make sure dynamicTemplates and defaultFields is not accessed in parallel
t.Lock()
defer t.Unlock()
dynamicTemplates = nil
defaultFields = nil
var err error
if len(t.config.AppendFields) > 0 {
cfgwarn.Experimental("append_fields is used.")
fields, err = appendFields(fields, t.config.AppendFields)
if err != nil {
return nil, err
}
}
// Start processing at the root
properties := common.MapStr{}
processor := Processor{EsVersion: t.esVersion}
if err := processor.Process(fields, "", properties); err != nil {
return nil, err
}
output := t.Generate(properties, dynamicTemplates)
return output, nil
}
// LoadFile loads the the template from the given file path
func (t *Template) LoadFile(file string) (common.MapStr, error) {
fields, err := common.LoadFieldsYaml(file)
if err != nil {
return nil, err
}
return t.load(fields)
}
// LoadBytes loads the the template from the given byte array
func (t *Template) LoadBytes(data []byte) (common.MapStr, error) {
fields, err := loadYamlByte(data)
if err != nil {
return nil, err
}
return t.load(fields)
}
// GetName returns the name of the template
func (t *Template) GetName() string {
return t.name
}
// GetPattern returns the pattern of the template
func (t *Template) GetPattern() string {
return t.pattern
}
// Generate generates the full template
// The default values are taken from the default variable.
func (t *Template) Generate(properties common.MapStr, dynamicTemplates []common.MapStr) common.MapStr {
// Add base dynamic template
var dynamicTemplateBase = common.MapStr{
"strings_as_keyword": common.MapStr{
"mapping": common.MapStr{
"ignore_above": 1024,
"type": "keyword",
},
"match_mapping_type": "string",
},
}
if t.esVersion.IsMajor(2) {
dynamicTemplateBase.Put("strings_as_keyword.mapping.type", "string")
dynamicTemplateBase.Put("strings_as_keyword.mapping.index", "not_analyzed")
}
dynamicTemplates = append(dynamicTemplates, dynamicTemplateBase)
indexSettings := common.MapStr{
"refresh_interval": "5s",
"mapping": common.MapStr{
"total_fields": common.MapStr{
"limit": defaultTotalFieldsLimit,
},
},
}
// number_of_routing shards is only supported for ES version >= 6.1
version61, _ := common.NewVersion("6.1.0")
if !t.esVersion.LessThan(version61) {
indexSettings.Put("number_of_routing_shards", defaultNumberOfRoutingShards)
}
if t.esVersion.IsMajor(7) {
defaultFields = append(defaultFields, "fields.*")
indexSettings.Put("query.default_field", defaultFields)
}
indexSettings.DeepUpdate(t.config.Settings.Index)
var mappingName string
if t.esVersion.Major >= 6 {
mappingName = "doc"
} else {
mappingName = "_default_"
}
// Load basic structure
basicStructure := common.MapStr{
"mappings": common.MapStr{
mappingName: common.MapStr{
"_meta": common.MapStr{
"version": t.beatVersion.String(),
},
"date_detection": defaultDateDetection,
"dynamic_templates": dynamicTemplates,
"properties": properties,
},
},
"order": 1,
"settings": common.MapStr{
"index": indexSettings,
},
}
if len(t.config.Settings.Source) > 0 {
key := fmt.Sprintf("mappings.%s._source", mappingName)
basicStructure.Put(key, t.config.Settings.Source)
}
// ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009
if t.esVersion.Major >= 6 {
basicStructure.Put("index_patterns", []string{t.GetPattern()})
} else {
basicStructure.Put("template", t.GetPattern())
}
if t.esVersion.IsMajor(2) {
basicStructure.Put("mappings._default_._all.norms.enabled", false)
}
return basicStructure
}
func appendFields(fields, appendFields common.Fields) (common.Fields, error) {
if len(appendFields) > 0 {
appendFieldKeys := appendFields.GetKeys()
// Append is only allowed to add fields, not overwrite
for _, key := range appendFieldKeys {
if fields.HasNode(key) {
return nil, fmt.Errorf("append_fields contains an already existing key: %s", key)
}
}
// Appends fields to existing fields
fields = append(fields, appendFields...)
}
return fields, nil
}
func loadYamlByte(data []byte) (common.Fields, error) {
var keys []common.Field
cfg, err := yaml.NewConfig(data)
if err != nil {
return nil, err
}
cfg.Unpack(&keys)
fields := common.Fields{}
for _, key := range keys {
fields = append(fields, key.Fields...)
}
return fields, nil
}