youtubebeat/vendor/github.com/elastic/beats/dev-tools/mage/pkgtypes.go

826 lines
22 KiB
Go

// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package mage
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
"reflect"
"runtime"
"strconv"
"strings"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"github.com/mitchellh/hashstructure"
"github.com/pkg/errors"
)
const (
// distributionsDir is the dir where packages are written.
distributionsDir = "build/distributions"
// packageStagingDir is the staging directory for any temporary files that
// need to be written to disk for inclusion in a package.
packageStagingDir = "build/package"
// defaultBinaryName specifies the output file for zip and tar.gz.
defaultBinaryName = "{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}"
)
// PackageType defines the file format of the package (e.g. zip, rpm, etc).
type PackageType int
// List of possible package types.
const (
RPM PackageType = iota + 1
Deb
Zip
TarGz
DMG
)
// OSPackageArgs define a set of package types to build for an operating
// system using the contained PackageSpec.
type OSPackageArgs struct {
OS string `yaml:"os"`
Types []PackageType `yaml:"types"`
Spec PackageSpec `yaml:"spec"`
}
// PackageSpec specifies package metadata and the contents of the package.
type PackageSpec struct {
Name string `yaml:"name,omitempty"`
ServiceName string `yaml:"service_name,omitempty"`
OS string `yaml:"os,omitempty"`
Arch string `yaml:"arch,omitempty"`
Vendor string `yaml:"vendor,omitempty"`
Snapshot bool `yaml:"snapshot"`
Version string `yaml:"version,omitempty"`
License string `yaml:"license,omitempty"`
URL string `yaml:"url,omitempty"`
Description string `yaml:"description,omitempty"`
PreInstallScript string `yaml:"pre_install_script,omitempty"`
PostInstallScript string `yaml:"post_install_script,omitempty"`
Files map[string]PackageFile `yaml:"files"`
OutputFile string `yaml:"output_file,omitempty"` // Optional
ExtraVars map[string]string `yaml:"extra_vars,omitempty"` // Optional
evalContext map[string]interface{}
packageDir string
localPreInstallScript string
localPostInstallScript string
}
// PackageFile represents a file or directory within a package.
type PackageFile struct {
Source string `yaml:"source,omitempty"` // Regular source file or directory.
Content string `yaml:"content,omitempty"` // Inline template string.
Template string `yaml:"template,omitempty"` // Input template file.
Target string `yaml:"target,omitempty"` // Target location in package. Relative paths are added to a package specific directory (e.g. metricbeat-7.0.0-linux-x86_64).
Mode os.FileMode `yaml:"mode,omitempty"` // Target mode for file. Does not apply when source is a directory.
Config bool `yaml:"config"` // Mark file as config in the package (deb and rpm only).
Dep func(PackageSpec) error `yaml:"-" hash:"-" json:"-"` // Dependency to invoke during Evaluate.
}
// OSArchNames defines the names of architectures for use in packages.
var OSArchNames = map[string]map[PackageType]map[string]string{
"windows": map[PackageType]map[string]string{
Zip: map[string]string{
"386": "x86",
"amd64": "x86_64",
},
},
"darwin": map[PackageType]map[string]string{
TarGz: map[string]string{
"386": "x86",
"amd64": "x86_64",
},
DMG: map[string]string{
"386": "x86",
"amd64": "x86_64",
},
},
"linux": map[PackageType]map[string]string{
RPM: map[string]string{
"386": "i686",
"amd64": "x86_64",
"armv7": "armhfp",
"arm64": "aarch64",
"mipsle": "mipsel",
"mips64le": "mips64el",
"ppc64": "ppc64",
"ppc64le": "ppc64le",
"s390x": "s390x",
},
// https://www.debian.org/ports/
Deb: map[string]string{
"386": "i386",
"amd64": "amd64",
"armv5": "armel",
"armv6": "armel",
"armv7": "armhf",
"arm64": "arm64",
"mips": "mips",
"mipsle": "mipsel",
"mips64le": "mips64el",
"ppc64le": "ppc64el",
"s390x": "s390x",
},
TarGz: map[string]string{
"386": "x86",
"amd64": "x86_64",
"armv5": "armv5",
"armv6": "armv6",
"armv7": "armv7",
"arm64": "arm64",
"mips": "mips",
"mipsle": "mipsel",
"mips64": "mips64",
"mips64le": "mips64el",
"ppc64": "ppc64",
"ppc64le": "ppc64le",
"s390x": "s390x",
},
},
}
// getOSArchName returns the architecture name to use in a package.
func getOSArchName(platform BuildPlatform, t PackageType) (string, error) {
names, found := OSArchNames[platform.GOOS()]
if !found {
return "", errors.Errorf("arch names for os=%v are not defined",
platform.GOOS())
}
archMap, found := names[t]
if !found {
return "", errors.Errorf("arch names for %v on os=%v are not defined",
t, platform.GOOS())
}
arch, found := archMap[platform.Arch()]
if !found {
return "", errors.Errorf("arch name associated with %v for %v on "+
"os=%v is not defined", platform.Arch(), t, platform.GOOS())
}
return arch, nil
}
// String returns the name of the package type.
func (typ PackageType) String() string {
switch typ {
case RPM:
return "rpm"
case Deb:
return "deb"
case Zip:
return "zip"
case TarGz:
return "tar.gz"
case DMG:
return "dmg"
default:
return "invalid"
}
}
// MarshalText returns the text representation of PackageType.
func (typ PackageType) MarshalText() ([]byte, error) {
return []byte(typ.String()), nil
}
// UnmarshalText returns a PackageType based on the given text.
func (typ *PackageType) UnmarshalText(text []byte) error {
switch strings.ToLower(string(text)) {
case "rpm":
*typ = RPM
case "deb":
*typ = Deb
case "tar.gz", "tgz", "targz":
*typ = TarGz
case "zip":
*typ = Zip
case "dmg":
*typ = DMG
default:
return errors.Errorf("unknown package type: %v", string(text))
}
return nil
}
// AddFileExtension returns a filename with the file extension added. If the
// filename already has the extension then it becomes a pass-through.
func (typ PackageType) AddFileExtension(file string) string {
ext := "." + strings.ToLower(typ.String())
if !strings.HasSuffix(file, ext) {
return file + ext
}
return file
}
// Build builds a package based on the provided spec.
func (typ PackageType) Build(spec PackageSpec) error {
switch typ {
case RPM:
return PackageRPM(spec)
case Deb:
return PackageDeb(spec)
case Zip:
return PackageZip(spec)
case TarGz:
return PackageTarGz(spec)
case DMG:
return PackageDMG(spec)
default:
return errors.Errorf("unknown package type: %v", typ)
}
}
// Clone returns a deep clone of the spec.
func (s PackageSpec) Clone() PackageSpec {
clone := s
clone.Files = make(map[string]PackageFile, len(s.Files))
for k, v := range s.Files {
clone.Files[k] = v
}
return clone
}
// ReplaceFile replaces an existing file defined in the spec. The target must
// exist other it will panic.
func (s PackageSpec) ReplaceFile(target string, file PackageFile) {
_, found := s.Files[target]
if !found {
panic(errors.Errorf("failed to ReplaceFile because target=%v does not exist", target))
}
s.Files[target] = file
}
// Expand expands a templated string using data from the spec.
func (s PackageSpec) Expand(in string, args ...map[string]interface{}) (string, error) {
return expandTemplate("inline", in, FuncMap,
EnvMap(append([]map[string]interface{}{s.evalContext, s.toMap()}, args...)...))
}
// MustExpand expands a templated string using data from the spec. It panics if
// an error occurs.
func (s PackageSpec) MustExpand(in string, args ...map[string]interface{}) string {
v, err := s.Expand(in, args...)
if err != nil {
panic(err)
}
return v
}
// ExpandFile expands a template file using data from the spec.
func (s PackageSpec) ExpandFile(src, dst string, args ...map[string]interface{}) error {
return expandFile(src, dst,
EnvMap(append([]map[string]interface{}{s.evalContext, s.toMap()}, args...)...))
}
// MustExpandFile expands a template file using data from the spec. It panics if
// an error occurs.
func (s PackageSpec) MustExpandFile(src, dst string, args ...map[string]interface{}) {
if err := s.ExpandFile(src, dst, args...); err != nil {
panic(err)
}
}
// Evaluate expands all variables used in the spec definition and writes any
// templated files used in the spec to disk. It panics if there is an error.
func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec {
args = append([]map[string]interface{}{s.toMap(), s.evalContext}, args...)
mustExpand := func(in string) string {
if in == "" {
return ""
}
return MustExpand(in, args...)
}
for k, v := range s.ExtraVars {
s.evalContext[k] = mustExpand(v)
}
s.Name = mustExpand(s.Name)
s.ServiceName = mustExpand(s.ServiceName)
s.OS = mustExpand(s.OS)
s.Arch = mustExpand(s.Arch)
s.Vendor = mustExpand(s.Vendor)
s.Version = mustExpand(s.Version)
s.License = mustExpand(s.License)
s.URL = mustExpand(s.URL)
s.Description = mustExpand(s.Description)
s.PreInstallScript = mustExpand(s.PreInstallScript)
s.PostInstallScript = mustExpand(s.PostInstallScript)
s.OutputFile = mustExpand(s.OutputFile)
if s.ServiceName == "" {
s.ServiceName = s.Name
}
if s.packageDir == "" {
outputFileName := filepath.Base(s.OutputFile)
if outputFileName != "." {
s.packageDir = filepath.Join(packageStagingDir, outputFileName)
} else {
s.packageDir = filepath.Join(packageStagingDir, strings.Join([]string{s.Name, s.OS, s.Arch, s.hash()}, "-"))
}
} else {
s.packageDir = filepath.Clean(mustExpand(s.packageDir))
}
if s.evalContext == nil {
s.evalContext = map[string]interface{}{}
}
s.evalContext["PackageDir"] = s.packageDir
evaluatedFiles := make(map[string]PackageFile, len(s.Files))
for target, f := range s.Files {
// Execute the dependency if it exists.
if f.Dep != nil {
if err := f.Dep(s); err != nil {
panic(errors.Wrapf(err, "failed executing package file dependency for target=%v", target))
}
}
f.Source = s.MustExpand(f.Source)
f.Template = s.MustExpand(f.Template)
f.Target = s.MustExpand(target)
target = f.Target
// Expand templates.
switch {
case f.Source != "":
case f.Content != "":
content, err := s.Expand(f.Content)
if err != nil {
panic(errors.Wrapf(err, "failed to expand content template for target=%v", target))
}
f.Source = filepath.Join(s.packageDir, filepath.Base(f.Target))
if err = ioutil.WriteFile(createDir(f.Source), []byte(content), 0644); err != nil {
panic(errors.Wrapf(err, "failed to write file containing content for target=%v", target))
}
case f.Template != "":
f.Source = filepath.Join(s.packageDir, filepath.Base(f.Template))
if err := s.ExpandFile(f.Template, createDir(f.Source)); err != nil {
panic(errors.Wrapf(err, "failed to expand template file for target=%v", target))
}
default:
panic(errors.Errorf("package file with target=%v must have either source, content, or template", target))
}
evaluatedFiles[f.Target] = f
}
// Replace the map instead of modifying the source.
s.Files = evaluatedFiles
if err := copyInstallScript(s, s.PreInstallScript, &s.localPreInstallScript); err != nil {
panic(err)
}
if err := copyInstallScript(s, s.PostInstallScript, &s.localPostInstallScript); err != nil {
panic(err)
}
return s
}
func copyInstallScript(spec PackageSpec, script string, local *string) error {
if script == "" {
return nil
}
*local = filepath.Join(spec.packageDir, "scripts", filepath.Base(script))
if filepath.Ext(*local) == ".tmpl" {
*local = strings.TrimSuffix(*local, ".tmpl")
}
if err := spec.ExpandFile(script, createDir(*local)); err != nil {
return errors.Wrap(err, "failed to copy install script to package dir")
}
if err := os.Chmod(*local, 0755); err != nil {
return errors.Wrap(err, "failed to chmod install script")
}
return nil
}
func (s PackageSpec) hash() string {
h, err := hashstructure.Hash(s, nil)
if err != nil {
panic(errors.Wrap(err, "failed to compute hash of spec"))
}
hash := strconv.FormatUint(h, 10)
if len(hash) > 10 {
hash = hash[0:10]
}
return hash
}
// toMap returns a map containing the exported field names and their values.
func (s PackageSpec) toMap() map[string]interface{} {
out := make(map[string]interface{})
v := reflect.ValueOf(s)
typ := v.Type()
for i := 0; i < v.NumField(); i++ {
structField := typ.Field(i)
if !structField.Anonymous && structField.PkgPath == "" {
out[structField.Name] = v.Field(i).Interface()
}
}
return out
}
// rootDir returns the name of the root directory contained inside of zip and
// tar.gz packages.
func (s PackageSpec) rootDir() string {
if s.OutputFile != "" {
return filepath.Base(s.OutputFile)
}
// NOTE: This uses .BeatName instead of .Name because we wanted the internal
// directory to not include "-oss".
return s.MustExpand("{{.BeatName}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}")
}
// PackageZip packages a zip file.
func PackageZip(spec PackageSpec) error {
// Create a buffer to write our archive to.
buf := new(bytes.Buffer)
// Create a new zip archive.
w := zip.NewWriter(buf)
baseDir := spec.rootDir()
// Add files to zip.
for _, pkgFile := range spec.Files {
if err := addFileToZip(w, baseDir, pkgFile); err != nil {
return errors.Wrapf(err, "failed adding file=%+v to zip", pkgFile)
}
}
if err := w.Close(); err != nil {
return err
}
// Output the zip file.
if spec.OutputFile == "" {
outputZip, err := spec.Expand(defaultBinaryName + ".zip")
if err != nil {
return err
}
spec.OutputFile = filepath.Join(distributionsDir, outputZip)
}
spec.OutputFile = Zip.AddFileExtension(spec.OutputFile)
// Write the zip file.
if err := ioutil.WriteFile(createDir(spec.OutputFile), buf.Bytes(), 0644); err != nil {
return errors.Wrap(err, "failed to write zip file")
}
// Any packages beginning with "tmp-" are temporary by nature so don't have
// them a .sha512 file.
if strings.HasPrefix(filepath.Base(spec.OutputFile), "tmp-") {
return nil
}
return errors.Wrap(CreateSHA512File(spec.OutputFile), "failed to create .sha512 file")
}
// PackageTarGz packages a gzipped tar file.
func PackageTarGz(spec PackageSpec) error {
// Create a buffer to write our archive to.
buf := new(bytes.Buffer)
// Create a new tar archive.
w := tar.NewWriter(buf)
baseDir := spec.rootDir()
// Add files to tar.
for _, pkgFile := range spec.Files {
if err := addFileToTar(w, baseDir, pkgFile); err != nil {
return errors.Wrapf(err, "failed adding file=%+v to tar", pkgFile)
}
}
if err := w.Close(); err != nil {
return err
}
// Output tar.gz to disk.
if spec.OutputFile == "" {
outputTarGz, err := spec.Expand(defaultBinaryName + ".tar.gz")
if err != nil {
return err
}
spec.OutputFile = filepath.Join(distributionsDir, outputTarGz)
}
spec.OutputFile = TarGz.AddFileExtension(spec.OutputFile)
// Open the output file.
log.Println("Creating output file at", spec.OutputFile)
outFile, err := os.Create(createDir(spec.OutputFile))
if err != nil {
return err
}
defer outFile.Close()
// Gzip compress the data.
gzWriter := gzip.NewWriter(outFile)
if _, err = gzWriter.Write(buf.Bytes()); err != nil {
return err
}
// Close and flush.
if err = gzWriter.Close(); err != nil {
return err
}
// Any packages beginning with "tmp-" are temporary by nature so don't have
// them a .sha512 file.
if strings.HasPrefix(filepath.Base(spec.OutputFile), "tmp-") {
return nil
}
return errors.Wrap(CreateSHA512File(spec.OutputFile), "failed to create .sha512 file")
}
// PackageDeb packages a deb file. This requires Docker to execute FPM.
func PackageDeb(spec PackageSpec) error {
return runFPM(spec, Deb)
}
// PackageRPM packages a RPM file. This requires Docker to execute FPM.
func PackageRPM(spec PackageSpec) error {
return runFPM(spec, RPM)
}
func runFPM(spec PackageSpec, packageType PackageType) error {
var fpmPackageType string
switch packageType {
case RPM, Deb:
fpmPackageType = packageType.String()
default:
return errors.Errorf("unsupported package type=%v for runFPM", fpmPackageType)
}
if err := HaveDocker(); err != nil {
return fmt.Errorf("packaging %v files requires docker: %v", fpmPackageType, err)
}
// Build a tar file as the input to FPM.
inputTar := filepath.Join(distributionsDir, "tmp-"+fpmPackageType+"-"+spec.rootDir()+"-"+spec.hash()+".tar.gz")
spec.OutputFile = inputTar
if err := PackageTarGz(spec); err != nil {
return err
}
defer os.Remove(inputTar)
outputFile, err := spec.Expand("{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.Arch}}")
if err != nil {
return err
}
spec.OutputFile = packageType.AddFileExtension(filepath.Join(distributionsDir, outputFile))
dockerRun := sh.RunCmd("docker", "run")
var args []string
args, err = addUidGidEnvArgs(args)
if err != nil {
return err
}
args = append(args,
"--rm",
"-w", "/app",
"-v", CWD()+":/app",
beatsFPMImage+":"+fpmVersion,
"fpm", "--force",
"--input-type", "tar",
"--output-type", fpmPackageType,
"--name", spec.ServiceName,
"--architecture", spec.Arch,
)
if spec.Version != "" {
args = append(args, "--version", spec.Version)
}
if spec.Vendor != "" {
args = append(args, "--vendor", spec.Vendor)
}
if spec.License != "" {
args = append(args, "--license", strings.Replace(spec.License, " ", "-", -1))
}
if spec.Description != "" {
args = append(args, "--description", spec.Description)
}
if spec.URL != "" {
args = append(args, "--url", spec.URL)
}
if spec.localPostInstallScript != "" {
args = append(args, "--after-install", spec.localPostInstallScript)
}
for _, pf := range spec.Files {
if pf.Config {
args = append(args, "--config-files", pf.Target)
}
}
args = append(args,
"-p", spec.OutputFile,
inputTar,
)
if err = dockerRun(args...); err != nil {
return errors.Wrap(err, "failed while running FPM in docker")
}
return errors.Wrap(CreateSHA512File(spec.OutputFile), "failed to create .sha512 file")
}
func addUidGidEnvArgs(args []string) ([]string, error) {
if runtime.GOOS == "windows" {
return args, nil
}
info, err := GetDockerInfo()
if err != nil {
return args, errors.Wrap(err, "failed to get docker info")
}
uid, gid := os.Getuid(), os.Getgid()
if info.IsBoot2Docker() {
// Boot2Docker mounts vboxfs using 1000:50.
uid, gid = 1000, 50
log.Printf("Boot2Docker is in use. Deploying workaround. "+
"Using UID=%d GID=%d", uid, gid)
}
return append(args,
"--env", "EXEC_UID="+strconv.Itoa(uid),
"--env", "EXEC_GID="+strconv.Itoa(gid),
), nil
}
// addFileToZip adds a file (or directory) to a zip archive.
func addFileToZip(ar *zip.Writer, baseDir string, pkgFile PackageFile) error {
return filepath.Walk(pkgFile.Source, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
header, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
if info.Mode().IsRegular() && pkgFile.Mode > 0 {
header.SetMode(pkgFile.Mode & os.ModePerm)
} else if info.IsDir() {
header.SetMode(0755)
}
if filepath.IsAbs(pkgFile.Target) {
baseDir = ""
}
relPath, err := filepath.Rel(pkgFile.Source, path)
if err != nil {
return err
}
header.Name = filepath.Join(baseDir, pkgFile.Target, relPath)
if info.IsDir() {
header.Name += string(filepath.Separator)
} else {
header.Method = zip.Deflate
}
if mg.Verbose() {
log.Println("Adding", header.Mode(), header.Name)
}
w, err := ar.CreateHeader(header)
if err != nil {
return err
}
if info.IsDir() {
return nil
}
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
if _, err = io.Copy(w, file); err != nil {
return err
}
return file.Close()
})
}
// addFileToTar adds a file (or directory) to a tar archive.
func addFileToTar(ar *tar.Writer, baseDir string, pkgFile PackageFile) error {
return filepath.Walk(pkgFile.Source, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
header, err := tar.FileInfoHeader(info, info.Name())
if err != nil {
return err
}
header.Uname, header.Gname = "root", "root"
header.Uid, header.Gid = 0, 0
if info.Mode().IsRegular() && pkgFile.Mode > 0 {
header.Mode = int64(pkgFile.Mode & os.ModePerm)
} else if info.IsDir() {
header.Mode = int64(0755)
}
if filepath.IsAbs(pkgFile.Target) {
baseDir = ""
}
relPath, err := filepath.Rel(pkgFile.Source, path)
if err != nil {
return err
}
header.Name = filepath.Join(baseDir, pkgFile.Target, relPath)
if info.IsDir() {
header.Name += string(filepath.Separator)
}
if mg.Verbose() {
log.Println("Adding", os.FileMode(header.Mode), header.Name)
}
if err := ar.WriteHeader(header); err != nil {
return err
}
if info.IsDir() {
return nil
}
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
if _, err = io.Copy(ar, file); err != nil {
return err
}
return file.Close()
})
}
// PackageDMG packages the Beat into a .dmg file containing an installer pkg
// and uninstaller app.
func PackageDMG(spec PackageSpec) error {
if runtime.GOOS != "darwin" {
return errors.New("packaging a dmg requires darwin")
}
b, err := newDMGBuilder(spec)
if err != nil {
return err
}
return b.Build()
}