youtubebeat/vendor/github.com/elastic/beats/libbeat/dashboards/es_loader.go

332 lines
8.8 KiB
Go
Raw Normal View History

2018-11-18 11:08:38 +01:00
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package dashboards
import (
"encoding/json"
"fmt"
"io/ioutil"
"path"
"path/filepath"
"strings"
"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/logp"
"github.com/elastic/beats/libbeat/outputs/elasticsearch"
)
type ElasticsearchLoader struct {
client *elasticsearch.Client
config *Config
version string
msgOutputter MessageOutputter
}
func NewElasticsearchLoader(cfg *common.Config, dashboardsConfig *Config, msgOutputter MessageOutputter) (*ElasticsearchLoader, error) {
if cfg == nil || !cfg.Enabled() {
return nil, fmt.Errorf("Elasticsearch output is not configured/enabled")
}
esClient, err := elasticsearch.NewConnectedClient(cfg)
if err != nil {
return nil, fmt.Errorf("Error creating Elasticsearch client: %v", err)
}
version := esClient.GetVersion()
loader := ElasticsearchLoader{
client: esClient,
config: dashboardsConfig,
version: version,
msgOutputter: msgOutputter,
}
loader.statusMsg("Initialize the Elasticsearch %s loader", version)
return &loader, nil
}
// CreateKibanaIndex creates the kibana index if it doesn't exists and sets
// some index properties which are needed as a workaround for:
// https://github.com/elastic/beats-dashboards/issues/94
func (loader ElasticsearchLoader) CreateKibanaIndex() error {
status, err := loader.client.IndexExists(loader.config.KibanaIndex)
if err != nil {
if status != 404 {
return err
}
_, _, err = loader.client.CreateIndex(loader.config.KibanaIndex, nil)
if err != nil {
return fmt.Errorf("Failed to create index: %v", err)
}
_, _, err = loader.client.CreateIndex(loader.config.KibanaIndex+"/_mapping/search",
common.MapStr{
"search": common.MapStr{
"properties": common.MapStr{
"hits": common.MapStr{
"type": "integer",
},
"version": common.MapStr{
"type": "integer",
},
},
},
})
if err != nil {
return fmt.Errorf("Failed to set the mapping: %v", err)
}
}
return nil
}
func (loader ElasticsearchLoader) ImportIndex(file string) error {
reader, err := ioutil.ReadFile(file)
if err != nil {
return err
}
var indexContent common.MapStr
err = json.Unmarshal(reader, &indexContent)
if err != nil {
return fmt.Errorf("fail to unmarshal index content: %v", err)
}
indexName, ok := indexContent["title"].(string)
if !ok {
return fmt.Errorf("Missing title in the index-pattern file at %s", file)
}
if loader.config.Index != "" {
// change index pattern name
loader.statusMsg("Change index in index-pattern %s", indexName)
indexContent["title"] = loader.config.Index
}
path := "/" + loader.config.KibanaIndex + "/index-pattern/" + indexName
if _, err = loader.client.LoadJSON(path, indexContent); err != nil {
return err
}
return nil
}
func (loader ElasticsearchLoader) importJSONFile(fileType string, file string) error {
path := "/" + loader.config.KibanaIndex + "/" + fileType
reader, err := ioutil.ReadFile(file)
if err != nil {
return fmt.Errorf("Failed to read %s. Error: %s", file, err)
}
var jsonContent map[string]interface{}
err = json.Unmarshal(reader, &jsonContent)
if err != nil {
return fmt.Errorf("fail to unmarshal json file: %v", err)
}
fileBase := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))
body, err := loader.client.LoadJSON(path+"/"+fileBase, jsonContent)
if err != nil {
return fmt.Errorf("Failed to load %s under %s/%s: %s. Response body: %s", file, path, fileBase, err, body)
}
return nil
}
func (loader ElasticsearchLoader) importPanelsFromDashboard(file string) (err error) {
// directory with the dashboards
dir := filepath.Dir(file)
// main directory with dashboard, search, visualizations directories
mainDir := filepath.Dir(dir)
reader, err := ioutil.ReadFile(file)
if err != nil {
return
}
type record struct {
Title string `json:"title"`
PanelsJSON string `json:"panelsJSON"`
}
type panel struct {
ID string `json:"id"`
Type string `json:"type"`
}
var jsonContent record
err = json.Unmarshal(reader, &jsonContent)
if err != nil {
return fmt.Errorf("fail to unmarshal json content: %v", err)
}
var widgets []panel
err = json.Unmarshal([]byte(jsonContent.PanelsJSON), &widgets)
if err != nil {
return fmt.Errorf("fail to unmarshal panels content: %v", err)
}
for _, widget := range widgets {
if widget.Type == "visualization" {
err = loader.importVisualization(path.Join(mainDir, "visualization", widget.ID+".json"))
if err != nil {
return err
}
} else if widget.Type == "search" {
err = loader.importSearch(path.Join(mainDir, "search", widget.ID+".json"))
if err != nil {
return err
}
} else {
loader.statusMsg("Widgets: %v", widgets)
return fmt.Errorf("Unknown panel type %s in %s", widget.Type, file)
}
}
return
}
func (loader ElasticsearchLoader) importVisualization(file string) error {
loader.statusMsg("Import visualization %s", file)
reader, err := ioutil.ReadFile(file)
if err != nil {
return err
}
var vizContent common.MapStr
err = json.Unmarshal(reader, &vizContent)
if err != nil {
return fmt.Errorf("fail to unmarshal visualization content %s: %v", file, err)
}
if loader.config.Index != "" {
if savedObject, ok := vizContent["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
vizContent["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(loader.config.Index, savedObject)
}
if visState, ok := vizContent["visState"].(string); ok {
vizContent["visState"] = ReplaceIndexInVisState(loader.config.Index, visState)
}
}
vizName := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))
path := "/" + loader.config.KibanaIndex + "/visualization/" + vizName
if _, err := loader.client.LoadJSON(path, vizContent); err != nil {
return err
}
return loader.importSearchFromVisualization(file)
}
func (loader ElasticsearchLoader) importSearch(file string) error {
reader, err := ioutil.ReadFile(file)
if err != nil {
return err
}
searchName := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))
var searchContent common.MapStr
err = json.Unmarshal(reader, &searchContent)
if err != nil {
return fmt.Errorf("fail to unmarshal search content %s: %v", searchName, err)
}
if loader.config.Index != "" {
// change index pattern name
if savedObject, ok := searchContent["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
searchContent["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(loader.config.Index, savedObject)
}
}
path := "/" + loader.config.KibanaIndex + "/search/" + searchName
loader.statusMsg("Import search %s", file)
if _, err = loader.client.LoadJSON(path, searchContent); err != nil {
return err
}
return nil
}
func (loader ElasticsearchLoader) importSearchFromVisualization(file string) error {
type record struct {
Title string `json:"title"`
SavedSearchID string `json:"savedSearchId"`
}
reader, err := ioutil.ReadFile(file)
if err != nil {
return nil
}
var jsonContent record
err = json.Unmarshal(reader, &jsonContent)
if err != nil {
return fmt.Errorf("fail to unmarshal the search content: %v", err)
}
id := jsonContent.SavedSearchID
if len(id) == 0 {
// no search used
return nil
}
// directory with the visualizations
dir := filepath.Dir(file)
// main directory
mainDir := filepath.Dir(dir)
searchFile := path.Join(mainDir, "search", id+".json")
if searchFile != "" {
// visualization depends on search
if err := loader.importSearch(searchFile); err != nil {
return err
}
}
return nil
}
func (loader ElasticsearchLoader) ImportDashboard(file string) error {
/* load dashboard */
err := loader.importJSONFile("dashboard", file)
if err != nil {
return err
}
/* load the visualizations and searches that depend on the dashboard */
return loader.importPanelsFromDashboard(file)
}
func (loader ElasticsearchLoader) Close() error {
return loader.client.Close()
}
func (loader ElasticsearchLoader) statusMsg(msg string, a ...interface{}) {
if loader.msgOutputter != nil {
loader.msgOutputter(msg, a...)
} else {
logp.Debug("dashboards", msg, a...)
}
}