youtubebeat/vendor/github.com/elastic/beats/libbeat/dashboards/importer.go

361 lines
8.6 KiB
Go
Raw Normal View History

2018-11-18 11:08:38 +01:00
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package dashboards
import (
"archive/zip"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
"strings"
errw "github.com/pkg/errors"
"github.com/elastic/beats/libbeat/common"
)
// ErrNotFound returned when we cannot find any dashboard to import.
type ErrNotFound struct {
ErrorString string
}
// Error returns the human readable error.
func (e *ErrNotFound) Error() string { return e.ErrorString }
func newErrNotFound(s string, a ...interface{}) *ErrNotFound {
return &ErrNotFound{fmt.Sprintf(s, a...)}
}
// MessageOutputter is a function type for injecting status logging
// into this module.
type MessageOutputter func(msg string, a ...interface{})
type Importer struct {
cfg *Config
version common.Version
loader Loader
}
type Loader interface {
ImportIndex(file string) error
ImportDashboard(file string) error
statusMsg(msg string, a ...interface{})
Close() error
}
func NewImporter(version common.Version, cfg *Config, loader Loader) (*Importer, error) {
// Current max version is 6
if version.Major > 6 {
version.Major = 6
}
return &Importer{
cfg: cfg,
version: version,
loader: loader,
}, nil
}
// Import imports the Kibana dashboards according to the configuration options.
func (imp Importer) Import() error {
if imp.cfg.URL != "" || imp.cfg.File != "" {
err := imp.ImportArchive()
if err != nil {
return errw.Wrap(err, "Error importing URL/file")
}
} else {
err := imp.ImportKibanaDir(imp.cfg.Dir)
if err != nil {
return errw.Wrapf(err, "Error importing directory %s", imp.cfg.Dir)
}
}
return nil
}
func (imp Importer) ImportDashboard(file string) error {
imp.loader.statusMsg("Import dashboard %s", file)
return imp.loader.ImportDashboard(file)
}
func (imp Importer) ImportFile(fileType string, file string) error {
imp.loader.statusMsg("Import %s from %s", fileType, file)
if fileType == "dashboard" {
return imp.loader.ImportDashboard(file)
} else if fileType == "index-pattern" {
return imp.loader.ImportIndex(file)
}
return fmt.Errorf("Unexpected file type %s", fileType)
}
func (imp Importer) ImportDir(dirType string, dir string) error {
imp.loader.statusMsg("Import directory %s", dir)
dir = path.Join(dir, dirType)
var errors []string
files, err := filepath.Glob(path.Join(dir, "*.json"))
if err != nil {
return fmt.Errorf("Failed to read directory %s. Error: %s", dir, err)
}
if len(files) == 0 {
return fmt.Errorf("The directory %s is empty, nothing to import", dir)
}
for _, file := range files {
err = imp.ImportFile(dirType, file)
if err != nil {
errors = append(errors, fmt.Sprintf(" error loading %s: %s", file, err))
}
}
if len(errors) > 0 {
return fmt.Errorf("Failed to load directory %s:\n%s", dir, strings.Join(errors, "\n"))
}
return nil
}
func (imp Importer) unzip(archive, target string) error {
imp.loader.statusMsg("Unzip archive %s", target)
reader, err := zip.OpenReader(archive)
if err != nil {
return err
}
defer reader.Close()
// Closure to close the files on each iteration
unzipFile := func(file *zip.File) error {
filePath := filepath.Join(target, file.Name)
// check that the resulting file path is indeed under target
// Note that Rel calls Clean.
relPath, err := filepath.Rel(target, filePath)
if err != nil {
return err
}
if strings.HasPrefix(filepath.ToSlash(relPath), "../") {
return fmt.Errorf("Zip file contains files outside of the target directory: %s", relPath)
}
if file.FileInfo().IsDir() {
return os.MkdirAll(filePath, file.Mode())
}
if err = os.MkdirAll(filepath.Dir(filePath), 0755); err != nil {
return fmt.Errorf("failed making directory for file %v: %v", filePath, err)
}
fileReader, err := file.Open()
if err != nil {
return err
}
defer fileReader.Close()
targetFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
if err != nil {
return err
}
defer targetFile.Close()
if _, err := io.Copy(targetFile, fileReader); err != nil {
return err
}
return nil
}
for _, file := range reader.File {
err := unzipFile(file)
if err != nil {
return err
}
}
return nil
}
func (imp Importer) ImportArchive() error {
var archive string
target, err := ioutil.TempDir("", "tmp")
if err != nil {
return fmt.Errorf("Failed to generate a temporary directory name: %v", err)
}
if err = os.MkdirAll(target, 0755); err != nil {
return fmt.Errorf("Failed to create a temporary directory %s: %v", target, err)
}
defer os.RemoveAll(target) // clean up
imp.loader.statusMsg("Created temporary directory %s", target)
if imp.cfg.File != "" {
archive = imp.cfg.File
} else if imp.cfg.URL != "" {
archive, err = imp.downloadFile(imp.cfg.URL, target)
if err != nil {
return fmt.Errorf("Failed to download file: %s. Error: %v", imp.cfg.URL, err)
}
} else {
return errors.New("No archive file or URL is set - please use -file or -url option")
}
err = imp.unzip(archive, target)
if err != nil {
return fmt.Errorf("Failed to unzip the archive: %s: %v", archive, err)
}
dirs, err := getDirectories(target)
if err != nil {
return err
}
if len(dirs) != 1 {
return fmt.Errorf("Too many directories under %s", target)
}
dirs, err = getDirectories(dirs[0])
if err != nil {
return err
}
for _, dir := range dirs {
imp.loader.statusMsg("Importing Kibana from %s", dir)
if imp.cfg.Beat == "" || filepath.Base(dir) == imp.cfg.Beat {
err = imp.ImportKibanaDir(dir)
if err != nil {
return err
}
}
}
return nil
}
func getDirectories(target string) ([]string, error) {
files, err := ioutil.ReadDir(target)
if err != nil {
return nil, err
}
var dirs []string
for _, file := range files {
if file.IsDir() {
dirs = append(dirs, filepath.Join(target, file.Name()))
}
}
return dirs, nil
}
func (imp Importer) downloadFile(url string, target string) (string, error) {
fileName := filepath.Base(url)
targetPath := path.Join(target, fileName)
imp.loader.statusMsg("Downloading %s", url)
// Create the file
out, err := os.Create(targetPath)
if err != nil {
return targetPath, err
}
defer out.Close()
// Get the data
resp, err := http.Get(url)
if err != nil {
return targetPath, err
}
if resp.StatusCode != 200 {
return targetPath, fmt.Errorf("Server returned: %s", resp.Status)
}
defer resp.Body.Close()
// Writer the body to file
_, err = io.Copy(out, resp.Body)
if err != nil {
return targetPath, err
}
return targetPath, nil
}
// import Kibana dashboards and index-pattern or only one of these
func (imp Importer) ImportKibanaDir(dir string) error {
var err error
versionPath := strconv.Itoa(imp.version.Major)
dir = path.Join(dir, versionPath)
imp.loader.statusMsg("Importing directory %v", dir)
if _, err := os.Stat(dir); err != nil {
return newErrNotFound("No directory %s", dir)
}
check := []string{}
if !imp.cfg.OnlyDashboards {
check = append(check, "index-pattern")
}
wantDashboards := false
if !imp.cfg.OnlyIndex {
check = append(check, "dashboard")
wantDashboards = true
}
types := []string{}
for _, c := range check {
if imp.subdirExists(dir, c) {
types = append(types, c)
}
}
if len(types) == 0 {
return newErrNotFound("The directory %s does not contain the %s subdirectory."+
" There is nothing to import into Kibana.", dir, strings.Join(check, " or "))
}
importDashboards := false
for _, t := range types {
err = imp.ImportDir(t, dir)
if err != nil {
return fmt.Errorf("Failed to import %s: %v", t, err)
}
if t == "dashboard" {
importDashboards = true
}
}
if wantDashboards && !importDashboards {
return newErrNotFound("No dashboards to import. Please make sure the %s directory "+
"contains a dashboard directory.", dir)
}
return nil
}
func (imp Importer) subdirExists(parent string, child string) bool {
if _, err := os.Stat(path.Join(parent, child)); err != nil {
return false
}
return true
}