// Licensed to Elasticsearch B.V. under one or more contributor // license agreements. See the NOTICE file distributed with // this work for additional information regarding copyright // ownership. Elasticsearch B.V. licenses this file to you under // the Apache License, Version 2.0 (the "License"); you may // not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. // +build mage package main import ( "context" "fmt" "log" "regexp" "strings" "time" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" "github.com/pkg/errors" "github.com/elastic/beats/dev-tools/mage" ) func init() { mage.BeatDescription = "Packetbeat analyzes network traffic and sends the data to Elasticsearch." } // Build builds the Beat binary. func Build() error { return mage.Build(mage.DefaultBuildArgs()) } // GolangCrossBuild build the Beat binary inside of the golang-builder. // Do not use directly, use crossBuild instead. func GolangCrossBuild() error { if dep, found := crossBuildDeps[mage.Platform.Name]; found { mg.Deps(dep) } params := mage.DefaultGolangCrossBuildArgs() if flags, found := libpcapLDFLAGS[mage.Platform.Name]; found { params.Env = map[string]string{ "CGO_LDFLAGS": flags, } } if flags, found := libpcapCFLAGS[mage.Platform.Name]; found { params.Env["CGO_CFLAGS"] = flags } return mage.GolangCrossBuild(params) } // BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon). func BuildGoDaemon() error { return mage.BuildGoDaemon() } // CrossBuild cross-builds the beat for all target platforms. func CrossBuild() error { mg.Deps(patchCGODirectives) defer undoPatchCGODirectives() // These Windows builds write temporary .s and .o files into the packetbeat // dir so they cannot be run in parallel. Changing to a different CWD does // not change where the temp files get written so that cannot be used as a // fix. if err := mage.CrossBuild(mage.ForPlatforms("windows"), mage.Serially()); err != nil { return err } return mage.CrossBuild(mage.ForPlatforms("!windows")) } // CrossBuildXPack cross-builds the beat with XPack for all target platforms. func CrossBuildXPack() error { mg.Deps(patchCGODirectives) defer undoPatchCGODirectives() // These Windows builds write temporary .s and .o files into the packetbeat // dir so they cannot be run in parallel. Changing to a different CWD does // not change where the temp files get written so that cannot be used as a // fix. if err := mage.CrossBuildXPack(mage.ForPlatforms("windows"), mage.Serially()); err != nil { return err } return mage.CrossBuildXPack(mage.ForPlatforms("!windows")) } // CrossBuildGoDaemon cross-builds the go-daemon binary using Docker. func CrossBuildGoDaemon() error { return mage.CrossBuildGoDaemon() } // Clean cleans all generated files and build artifacts. func Clean() error { return mage.Clean() } // Package packages the Beat for distribution. // Use SNAPSHOT=true to build snapshots. // Use PLATFORMS to control the target platforms. func Package() { start := time.Now() defer func() { fmt.Println("package ran for", time.Since(start)) }() mage.UseElasticBeatPackaging() customizePackaging() mg.Deps(Update) mg.Deps(CrossBuild, CrossBuildXPack, CrossBuildGoDaemon) mg.SerialDeps(mage.Package, TestPackages) } // TestPackages tests the generated packages (i.e. file modes, owners, groups). func TestPackages() error { return mage.TestPackages() } // Update updates the generated files (aka make update). func Update() error { return sh.Run("make", "update") } // Fields generates a fields.yml for the Beat. func Fields() error { return mage.GenerateFieldsYAML("protos") } // GoTestUnit executes the Go unit tests. // Use TEST_COVERAGE=true to enable code coverage profiling. // Use RACE_DETECTOR=true to enable the race detector. func GoTestUnit(ctx context.Context) error { return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs()) } // GoTestIntegration executes the Go integration tests. // Use TEST_COVERAGE=true to enable code coverage profiling. // Use RACE_DETECTOR=true to enable the race detector. func GoTestIntegration(ctx context.Context) error { return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) } // ----------------------------------------------------------------------------- // Customizations specific to Packetbeat. // - Config file contains an OS specific device name (affects darwin, windows). // - Must compile libpcap or winpcap during cross-compilation. // - On Linux libpcap is statically linked. Darwin and Windows are dynamic. const ( libpcapURL = "https://s3.amazonaws.com/beats-files/deps/libpcap-1.8.1.tar.gz" libpcapSHA256 = "673dbc69fdc3f5a86fb5759ab19899039a8e5e6c631749e48dcd9c6f0c83541e" ) const ( linuxPcapLDFLAGS = "-L/libpcap/libpcap-1.8.1 -lpcap" linuxPcapCFLAGS = "-I /libpcap/libpcap-1.8.1" ) var libpcapLDFLAGS = map[string]string{ "linux/386": linuxPcapLDFLAGS, "linux/amd64": linuxPcapLDFLAGS, "linux/arm64": linuxPcapLDFLAGS, "linux/armv5": linuxPcapLDFLAGS, "linux/armv6": linuxPcapLDFLAGS, "linux/armv7": linuxPcapLDFLAGS, "linux/mips": linuxPcapLDFLAGS, "linux/mipsle": linuxPcapLDFLAGS, "linux/mips64": linuxPcapLDFLAGS, "linux/mips64le": linuxPcapLDFLAGS, "linux/ppc64le": linuxPcapLDFLAGS, "linux/s390x": linuxPcapLDFLAGS, "darwin/amd64": "-lpcap", "windows/amd64": "-L /libpcap/win/WpdPack/Lib/x64 -lwpcap", "windows/386": "-L /libpcap/win/WpdPack/Lib -lwpcap", } var libpcapCFLAGS = map[string]string{ "linux/386": linuxPcapCFLAGS, "linux/amd64": linuxPcapCFLAGS, "linux/arm64": linuxPcapCFLAGS, "linux/armv5": linuxPcapCFLAGS, "linux/armv6": linuxPcapCFLAGS, "linux/armv7": linuxPcapCFLAGS, "linux/mips": linuxPcapCFLAGS, "linux/mipsle": linuxPcapCFLAGS, "linux/mips64": linuxPcapCFLAGS, "linux/mips64le": linuxPcapCFLAGS, "linux/ppc64le": linuxPcapCFLAGS, "linux/s390x": linuxPcapCFLAGS, "windows/amd64": "-I /libpcap/win/WpdPack/Include", "windows/386": "-I /libpcap/win/WpdPack/Include", } var crossBuildDeps = map[string]func() error{ "linux/386": buildLibpcapLinux386, "linux/amd64": buildLibpcapLinuxAMD64, "linux/arm64": buildLibpcapLinuxARM64, "linux/armv5": buildLibpcapLinuxARMv5, "linux/armv6": buildLibpcapLinuxARMv6, "linux/armv7": buildLibpcapLinuxARMv7, "linux/mips": buildLibpcapLinuxMIPS, "linux/mipsle": buildLibpcapLinuxMIPSLE, "linux/mips64": buildLibpcapLinuxMIPS64, "linux/mips64le": buildLibpcapLinuxMIPS64LE, "linux/ppc64le": buildLibpcapLinuxPPC64LE, "linux/s390x": buildLibpcapLinuxS390x, "windows/amd64": installLibpcapWindowsAMD64, "windows/386": installLibpcapWindows386, } // buildLibpcapFromSource builds libpcap from source because the library needs // to be compiled with -fPIC. // See https://github.com/elastic/beats/pull/4217. func buildLibpcapFromSource(params map[string]string) error { tarFile, err := mage.DownloadFile(libpcapURL, "/libpcap") if err != nil { return errors.Wrap(err, "failed to download libpcap source") } if err = mage.VerifySHA256(tarFile, libpcapSHA256); err != nil { return err } if err = mage.Extract(tarFile, "/libpcap"); err != nil { return errors.Wrap(err, "failed to extract libpcap") } var configureArgs []string for k, v := range params { if strings.HasPrefix(k, "-") { delete(params, k) configureArgs = append(configureArgs, k+"="+v) } } // Use sh -c here because sh.Run does not expose a way to change the CWD. // This command only runs in Linux so this is fine. return sh.RunWith(params, "sh", "-c", "cd /libpcap/libpcap-1.8.1 && "+ "./configure --enable-usb=no --enable-bluetooth=no --enable-dbus=no "+strings.Join(configureArgs, " ")+"&& "+ "make") } func buildLibpcapLinux386() error { return buildLibpcapFromSource(map[string]string{ "CFLAGS": "-m32", "LDFLAGS": "-m32", }) } func buildLibpcapLinuxAMD64() error { return buildLibpcapFromSource(map[string]string{}) } func buildLibpcapLinuxARM64() error { return buildLibpcapFromSource(map[string]string{ "--host": "aarch64-unknown-linux-gnu", "--with-pcap": "linux", }) } func buildLibpcapLinuxARMv5() error { return buildLibpcapFromSource(map[string]string{ "--host": "arm-linux-gnueabi", "--with-pcap": "linux", }) } func buildLibpcapLinuxARMv6() error { return buildLibpcapFromSource(map[string]string{ "--host": "arm-linux-gnueabi", "--with-pcap": "linux", }) } func buildLibpcapLinuxARMv7() error { return buildLibpcapFromSource(map[string]string{ "--host": "arm-linux-gnueabihf", "--with-pcap": "linux", }) } func buildLibpcapLinuxMIPS() error { return buildLibpcapFromSource(map[string]string{ "--host": "mips-unknown-linux-gnu", "--with-pcap": "linux", }) } func buildLibpcapLinuxMIPSLE() error { return buildLibpcapFromSource(map[string]string{ "--host": "mipsle-unknown-linux-gnu", "--with-pcap": "linux", }) } func buildLibpcapLinuxMIPS64() error { return buildLibpcapFromSource(map[string]string{ "--host": "mips64-unknown-linux-gnu", "--with-pcap": "linux", }) } func buildLibpcapLinuxMIPS64LE() error { return buildLibpcapFromSource(map[string]string{ "--host": "mips64le-unknown-linux-gnu", "--with-pcap": "linux", }) } func buildLibpcapLinuxPPC64LE() error { return buildLibpcapFromSource(map[string]string{ "--host": "powerpc64le-linux-gnu", "--with-pcap": "linux", }) } func buildLibpcapLinuxS390x() error { return buildLibpcapFromSource(map[string]string{ "--host": "s390x-ibm-linux-gnu", "--with-pcap": "linux", }) } func installLibpcapWindowsAMD64() error { mg.SerialDeps(installWinpcap, generateWin64StaticWinpcap) return nil } func installLibpcapWindows386() error { return installWinpcap() } func installWinpcap() error { log.Println("Install Winpcap") const wpdpackURL = "https://www.winpcap.org/install/bin/WpdPack_4_1_2.zip" winpcapZip, err := mage.DownloadFile(wpdpackURL, "/") if err != nil { return err } if err = mage.Extract(winpcapZip, "/libpcap/win"); err != nil { return err } return nil } func generateWin64StaticWinpcap() error { log.Println(">> Generating 64-bit winpcap static lib") // Notes: We are using absolute path to make sure the files // are available for x-pack build. // Ref: https://github.com/elastic/beats/issues/1259 defer mage.DockerChown(mage.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib")) return mage.RunCmds( // Requires mingw-w64-tools. []string{"gendef", mage.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.dll")}, []string{"mv", "wpcap.def", mage.MustExpand("{{ elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.def")}, []string{"x86_64-w64-mingw32-dlltool", "--as-flags=--64", "-m", "i386:x86-64", "-k", "--output-lib", "/libpcap/win/WpdPack/Lib/x64/libwpcap.a", "--input-def", mage.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.def")}, ) } var pcapGoFile = mage.MustExpand("{{elastic_beats_dir}}/vendor/github.com/tsg/gopacket/pcap/pcap.go") var cgoDirectiveRegex = regexp.MustCompile(`(?m)#cgo .*(?:LDFLAGS|CFLAGS).*$`) func patchCGODirectives() error { // cgo directives do not support GOARM tags so we will clear the tags // and set them via CGO_LDFLAGS and CGO_CFLAGS. // Ref: https://github.com/golang/go/issues/7211 log.Println("Patching", pcapGoFile, cgoDirectiveRegex.String()) return mage.FindReplace(pcapGoFile, cgoDirectiveRegex, "") } func undoPatchCGODirectives() error { return sh.Run("git", "checkout", pcapGoFile) } // customizePackaging modifies the device in the configuration files based on // the target OS. func customizePackaging() { var ( defaultDevice = map[string]string{ "darwin": "en0", "windows": "0", } configYml = mage.PackageFile{ Mode: 0600, Source: "{{.PackageDir}}/{{.BeatName}}.yml", Config: true, Dep: func(spec mage.PackageSpec) error { if err := mage.Copy("packetbeat.yml", spec.MustExpand("{{.PackageDir}}/packetbeat.yml")); err != nil { return errors.Wrap(err, "failed to copy config") } return mage.FindReplace( spec.MustExpand("{{.PackageDir}}/packetbeat.yml"), regexp.MustCompile(`device: any`), "device: "+defaultDevice[spec.OS]) }, } referenceConfigYml = mage.PackageFile{ Mode: 0644, Source: "{{.PackageDir}}/{{.BeatName}}.reference.yml", Dep: func(spec mage.PackageSpec) error { if err := mage.Copy("packetbeat.yml", spec.MustExpand("{{.PackageDir}}/packetbeat.reference.yml")); err != nil { return errors.Wrap(err, "failed to copy config") } return mage.FindReplace( spec.MustExpand("{{.PackageDir}}/packetbeat.reference.yml"), regexp.MustCompile(`device: any`), "device: "+defaultDevice[spec.OS]) }, } ) for _, args := range mage.Packages { switch args.OS { case "windows", "darwin": if args.Types[0] == mage.DMG { args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.yml", configYml) args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.reference.yml", referenceConfigYml) continue } args.Spec.ReplaceFile("{{.BeatName}}.yml", configYml) args.Spec.ReplaceFile("{{.BeatName}}.reference.yml", referenceConfigYml) } } }