332 lines
8.8 KiB
Go
332 lines
8.8 KiB
Go
// Licensed to Elasticsearch B.V. under one or more contributor
|
|
// license agreements. See the NOTICE file distributed with
|
|
// this work for additional information regarding copyright
|
|
// ownership. Elasticsearch B.V. licenses this file to you under
|
|
// the Apache License, Version 2.0 (the "License"); you may
|
|
// not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
package dashboards
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"path"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/elastic/beats/libbeat/common"
|
|
"github.com/elastic/beats/libbeat/logp"
|
|
"github.com/elastic/beats/libbeat/outputs/elasticsearch"
|
|
)
|
|
|
|
type ElasticsearchLoader struct {
|
|
client *elasticsearch.Client
|
|
config *Config
|
|
version string
|
|
msgOutputter MessageOutputter
|
|
}
|
|
|
|
func NewElasticsearchLoader(cfg *common.Config, dashboardsConfig *Config, msgOutputter MessageOutputter) (*ElasticsearchLoader, error) {
|
|
if cfg == nil || !cfg.Enabled() {
|
|
return nil, fmt.Errorf("Elasticsearch output is not configured/enabled")
|
|
}
|
|
|
|
esClient, err := elasticsearch.NewConnectedClient(cfg)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Error creating Elasticsearch client: %v", err)
|
|
}
|
|
|
|
version := esClient.GetVersion()
|
|
|
|
loader := ElasticsearchLoader{
|
|
client: esClient,
|
|
config: dashboardsConfig,
|
|
version: version,
|
|
msgOutputter: msgOutputter,
|
|
}
|
|
|
|
loader.statusMsg("Initialize the Elasticsearch %s loader", version)
|
|
|
|
return &loader, nil
|
|
}
|
|
|
|
// CreateKibanaIndex creates the kibana index if it doesn't exists and sets
|
|
// some index properties which are needed as a workaround for:
|
|
// https://github.com/elastic/beats-dashboards/issues/94
|
|
func (loader ElasticsearchLoader) CreateKibanaIndex() error {
|
|
status, err := loader.client.IndexExists(loader.config.KibanaIndex)
|
|
|
|
if err != nil {
|
|
if status != 404 {
|
|
return err
|
|
}
|
|
|
|
_, _, err = loader.client.CreateIndex(loader.config.KibanaIndex, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to create index: %v", err)
|
|
}
|
|
|
|
_, _, err = loader.client.CreateIndex(loader.config.KibanaIndex+"/_mapping/search",
|
|
common.MapStr{
|
|
"search": common.MapStr{
|
|
"properties": common.MapStr{
|
|
"hits": common.MapStr{
|
|
"type": "integer",
|
|
},
|
|
"version": common.MapStr{
|
|
"type": "integer",
|
|
},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to set the mapping: %v", err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (loader ElasticsearchLoader) ImportIndex(file string) error {
|
|
reader, err := ioutil.ReadFile(file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var indexContent common.MapStr
|
|
err = json.Unmarshal(reader, &indexContent)
|
|
if err != nil {
|
|
return fmt.Errorf("fail to unmarshal index content: %v", err)
|
|
}
|
|
|
|
indexName, ok := indexContent["title"].(string)
|
|
if !ok {
|
|
return fmt.Errorf("Missing title in the index-pattern file at %s", file)
|
|
}
|
|
|
|
if loader.config.Index != "" {
|
|
// change index pattern name
|
|
loader.statusMsg("Change index in index-pattern %s", indexName)
|
|
indexContent["title"] = loader.config.Index
|
|
}
|
|
|
|
path := "/" + loader.config.KibanaIndex + "/index-pattern/" + indexName
|
|
|
|
if _, err = loader.client.LoadJSON(path, indexContent); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (loader ElasticsearchLoader) importJSONFile(fileType string, file string) error {
|
|
path := "/" + loader.config.KibanaIndex + "/" + fileType
|
|
|
|
reader, err := ioutil.ReadFile(file)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to read %s. Error: %s", file, err)
|
|
}
|
|
var jsonContent map[string]interface{}
|
|
err = json.Unmarshal(reader, &jsonContent)
|
|
if err != nil {
|
|
return fmt.Errorf("fail to unmarshal json file: %v", err)
|
|
}
|
|
|
|
fileBase := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))
|
|
|
|
body, err := loader.client.LoadJSON(path+"/"+fileBase, jsonContent)
|
|
if err != nil {
|
|
return fmt.Errorf("Failed to load %s under %s/%s: %s. Response body: %s", file, path, fileBase, err, body)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (loader ElasticsearchLoader) importPanelsFromDashboard(file string) (err error) {
|
|
// directory with the dashboards
|
|
dir := filepath.Dir(file)
|
|
|
|
// main directory with dashboard, search, visualizations directories
|
|
mainDir := filepath.Dir(dir)
|
|
|
|
reader, err := ioutil.ReadFile(file)
|
|
if err != nil {
|
|
return
|
|
}
|
|
type record struct {
|
|
Title string `json:"title"`
|
|
PanelsJSON string `json:"panelsJSON"`
|
|
}
|
|
type panel struct {
|
|
ID string `json:"id"`
|
|
Type string `json:"type"`
|
|
}
|
|
|
|
var jsonContent record
|
|
err = json.Unmarshal(reader, &jsonContent)
|
|
if err != nil {
|
|
return fmt.Errorf("fail to unmarshal json content: %v", err)
|
|
}
|
|
|
|
var widgets []panel
|
|
err = json.Unmarshal([]byte(jsonContent.PanelsJSON), &widgets)
|
|
if err != nil {
|
|
return fmt.Errorf("fail to unmarshal panels content: %v", err)
|
|
}
|
|
|
|
for _, widget := range widgets {
|
|
if widget.Type == "visualization" {
|
|
err = loader.importVisualization(path.Join(mainDir, "visualization", widget.ID+".json"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
} else if widget.Type == "search" {
|
|
err = loader.importSearch(path.Join(mainDir, "search", widget.ID+".json"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
loader.statusMsg("Widgets: %v", widgets)
|
|
return fmt.Errorf("Unknown panel type %s in %s", widget.Type, file)
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
func (loader ElasticsearchLoader) importVisualization(file string) error {
|
|
loader.statusMsg("Import visualization %s", file)
|
|
reader, err := ioutil.ReadFile(file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var vizContent common.MapStr
|
|
err = json.Unmarshal(reader, &vizContent)
|
|
if err != nil {
|
|
return fmt.Errorf("fail to unmarshal visualization content %s: %v", file, err)
|
|
}
|
|
|
|
if loader.config.Index != "" {
|
|
if savedObject, ok := vizContent["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
|
|
vizContent["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(loader.config.Index, savedObject)
|
|
}
|
|
|
|
if visState, ok := vizContent["visState"].(string); ok {
|
|
vizContent["visState"] = ReplaceIndexInVisState(loader.config.Index, visState)
|
|
}
|
|
}
|
|
|
|
vizName := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))
|
|
path := "/" + loader.config.KibanaIndex + "/visualization/" + vizName
|
|
if _, err := loader.client.LoadJSON(path, vizContent); err != nil {
|
|
return err
|
|
}
|
|
|
|
return loader.importSearchFromVisualization(file)
|
|
}
|
|
|
|
func (loader ElasticsearchLoader) importSearch(file string) error {
|
|
reader, err := ioutil.ReadFile(file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
searchName := strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))
|
|
|
|
var searchContent common.MapStr
|
|
err = json.Unmarshal(reader, &searchContent)
|
|
if err != nil {
|
|
return fmt.Errorf("fail to unmarshal search content %s: %v", searchName, err)
|
|
}
|
|
|
|
if loader.config.Index != "" {
|
|
|
|
// change index pattern name
|
|
if savedObject, ok := searchContent["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
|
|
|
|
searchContent["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(loader.config.Index, savedObject)
|
|
|
|
}
|
|
}
|
|
|
|
path := "/" + loader.config.KibanaIndex + "/search/" + searchName
|
|
loader.statusMsg("Import search %s", file)
|
|
|
|
if _, err = loader.client.LoadJSON(path, searchContent); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (loader ElasticsearchLoader) importSearchFromVisualization(file string) error {
|
|
type record struct {
|
|
Title string `json:"title"`
|
|
SavedSearchID string `json:"savedSearchId"`
|
|
}
|
|
|
|
reader, err := ioutil.ReadFile(file)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
var jsonContent record
|
|
err = json.Unmarshal(reader, &jsonContent)
|
|
if err != nil {
|
|
return fmt.Errorf("fail to unmarshal the search content: %v", err)
|
|
}
|
|
|
|
id := jsonContent.SavedSearchID
|
|
if len(id) == 0 {
|
|
// no search used
|
|
return nil
|
|
}
|
|
|
|
// directory with the visualizations
|
|
dir := filepath.Dir(file)
|
|
|
|
// main directory
|
|
mainDir := filepath.Dir(dir)
|
|
|
|
searchFile := path.Join(mainDir, "search", id+".json")
|
|
|
|
if searchFile != "" {
|
|
// visualization depends on search
|
|
if err := loader.importSearch(searchFile); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (loader ElasticsearchLoader) ImportDashboard(file string) error {
|
|
/* load dashboard */
|
|
err := loader.importJSONFile("dashboard", file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
/* load the visualizations and searches that depend on the dashboard */
|
|
return loader.importPanelsFromDashboard(file)
|
|
}
|
|
|
|
func (loader ElasticsearchLoader) Close() error {
|
|
return loader.client.Close()
|
|
}
|
|
|
|
func (loader ElasticsearchLoader) statusMsg(msg string, a ...interface{}) {
|
|
if loader.msgOutputter != nil {
|
|
loader.msgOutputter(msg, a...)
|
|
} else {
|
|
logp.Debug("dashboards", msg, a...)
|
|
}
|
|
}
|