428 lines
11 KiB
Go
428 lines
11 KiB
Go
|
// Licensed to Elasticsearch B.V. under one or more contributor
|
||
|
|
// license agreements. See the NOTICE file distributed with
|
||
|
|
// this work for additional information regarding copyright
|
||
|
|
// ownership. Elasticsearch B.V. licenses this file to you under
|
||
|
|
// the Apache License, Version 2.0 (the "License"); you may
|
||
|
|
// not use this file except in compliance with the License.
|
||
|
|
// You may obtain a copy of the License at
|
||
|
|
//
|
||
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
//
|
||
|
|
// Unless required by applicable law or agreed to in writing,
|
||
|
|
// software distributed under the License is distributed on an
|
||
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||
|
|
// KIND, either express or implied. See the License for the
|
||
|
|
// specific language governing permissions and limitations
|
||
|
|
// under the License.
|
||
|
|
|
||
|
|
package mongodb
|
||
|
|
|
||
|
|
import (
|
||
|
|
"fmt"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/elastic/beats/libbeat/beat"
|
||
|
|
"github.com/elastic/beats/libbeat/common"
|
||
|
|
"github.com/elastic/beats/libbeat/logp"
|
||
|
|
"github.com/elastic/beats/libbeat/monitoring"
|
||
|
|
"github.com/elastic/beats/packetbeat/procs"
|
||
|
|
"github.com/elastic/beats/packetbeat/protos"
|
||
|
|
"github.com/elastic/beats/packetbeat/protos/tcp"
|
||
|
|
)
|
||
|
|
|
||
|
|
var debugf = logp.MakeDebug("mongodb")
|
||
|
|
|
||
|
|
type mongodbPlugin struct {
|
||
|
|
// config
|
||
|
|
ports []int
|
||
|
|
sendRequest bool
|
||
|
|
sendResponse bool
|
||
|
|
maxDocs int
|
||
|
|
maxDocLength int
|
||
|
|
|
||
|
|
requests *common.Cache
|
||
|
|
responses *common.Cache
|
||
|
|
transactionTimeout time.Duration
|
||
|
|
|
||
|
|
results protos.Reporter
|
||
|
|
}
|
||
|
|
|
||
|
|
type transactionKey struct {
|
||
|
|
tcp common.HashableTCPTuple
|
||
|
|
id int
|
||
|
|
}
|
||
|
|
|
||
|
|
var (
|
||
|
|
unmatchedRequests = monitoring.NewInt(nil, "mongodb.unmatched_requests")
|
||
|
|
)
|
||
|
|
|
||
|
|
func init() {
|
||
|
|
protos.Register("mongodb", New)
|
||
|
|
}
|
||
|
|
|
||
|
|
func New(
|
||
|
|
testMode bool,
|
||
|
|
results protos.Reporter,
|
||
|
|
cfg *common.Config,
|
||
|
|
) (protos.Plugin, error) {
|
||
|
|
p := &mongodbPlugin{}
|
||
|
|
config := defaultConfig
|
||
|
|
if !testMode {
|
||
|
|
if err := cfg.Unpack(&config); err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if err := p.init(results, &config); err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
return p, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (mongodb *mongodbPlugin) init(results protos.Reporter, config *mongodbConfig) error {
|
||
|
|
debugf("Init a MongoDB protocol parser")
|
||
|
|
mongodb.setFromConfig(config)
|
||
|
|
|
||
|
|
mongodb.requests = common.NewCache(
|
||
|
|
mongodb.transactionTimeout,
|
||
|
|
protos.DefaultTransactionHashSize)
|
||
|
|
mongodb.requests.StartJanitor(mongodb.transactionTimeout)
|
||
|
|
mongodb.responses = common.NewCache(
|
||
|
|
mongodb.transactionTimeout,
|
||
|
|
protos.DefaultTransactionHashSize)
|
||
|
|
mongodb.responses.StartJanitor(mongodb.transactionTimeout)
|
||
|
|
mongodb.results = results
|
||
|
|
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (mongodb *mongodbPlugin) setFromConfig(config *mongodbConfig) {
|
||
|
|
mongodb.ports = config.Ports
|
||
|
|
mongodb.sendRequest = config.SendRequest
|
||
|
|
mongodb.sendResponse = config.SendResponse
|
||
|
|
mongodb.maxDocs = config.MaxDocs
|
||
|
|
mongodb.maxDocLength = config.MaxDocLength
|
||
|
|
mongodb.transactionTimeout = config.TransactionTimeout
|
||
|
|
}
|
||
|
|
|
||
|
|
func (mongodb *mongodbPlugin) GetPorts() []int {
|
||
|
|
return mongodb.ports
|
||
|
|
}
|
||
|
|
|
||
|
|
func (mongodb *mongodbPlugin) ConnectionTimeout() time.Duration {
|
||
|
|
return mongodb.transactionTimeout
|
||
|
|
}
|
||
|
|
|
||
|
|
func (mongodb *mongodbPlugin) Parse(
|
||
|
|
pkt *protos.Packet,
|
||
|
|
tcptuple *common.TCPTuple,
|
||
|
|
dir uint8,
|
||
|
|
private protos.ProtocolData,
|
||
|
|
) protos.ProtocolData {
|
||
|
|
defer logp.Recover("ParseMongodb exception")
|
||
|
|
debugf("Parse method triggered")
|
||
|
|
|
||
|
|
conn := ensureMongodbConnection(private)
|
||
|
|
conn = mongodb.doParse(conn, pkt, tcptuple, dir)
|
||
|
|
if conn == nil {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
return conn
|
||
|
|
}
|
||
|
|
|
||
|
|
func ensureMongodbConnection(private protos.ProtocolData) *mongodbConnectionData {
|
||
|
|
if private == nil {
|
||
|
|
return &mongodbConnectionData{}
|
||
|
|
}
|
||
|
|
|
||
|
|
priv, ok := private.(*mongodbConnectionData)
|
||
|
|
if !ok {
|
||
|
|
logp.Warn("mongodb connection data type error, create new one")
|
||
|
|
return &mongodbConnectionData{}
|
||
|
|
}
|
||
|
|
if priv == nil {
|
||
|
|
debugf("Unexpected: mongodb connection data not set, create new one")
|
||
|
|
return &mongodbConnectionData{}
|
||
|
|
}
|
||
|
|
|
||
|
|
return priv
|
||
|
|
}
|
||
|
|
|
||
|
|
func (mongodb *mongodbPlugin) doParse(
|
||
|
|
conn *mongodbConnectionData,
|
||
|
|
pkt *protos.Packet,
|
||
|
|
tcptuple *common.TCPTuple,
|
||
|
|
dir uint8,
|
||
|
|
) *mongodbConnectionData {
|
||
|
|
st := conn.streams[dir]
|
||
|
|
if st == nil {
|
||
|
|
st = newStream(pkt, tcptuple)
|
||
|
|
conn.streams[dir] = st
|
||
|
|
debugf("new stream: %p (dir=%v, len=%v)", st, dir, len(pkt.Payload))
|
||
|
|
} else {
|
||
|
|
// concatenate bytes
|
||
|
|
st.data = append(st.data, pkt.Payload...)
|
||
|
|
if len(st.data) > tcp.TCPMaxDataInStream {
|
||
|
|
debugf("Stream data too large, dropping TCP stream")
|
||
|
|
conn.streams[dir] = nil
|
||
|
|
return conn
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
for len(st.data) > 0 {
|
||
|
|
if st.message == nil {
|
||
|
|
st.message = &mongodbMessage{ts: pkt.Ts}
|
||
|
|
}
|
||
|
|
|
||
|
|
ok, complete := mongodbMessageParser(st)
|
||
|
|
if !ok {
|
||
|
|
// drop this tcp stream. Will retry parsing with the next
|
||
|
|
// segment in it
|
||
|
|
conn.streams[dir] = nil
|
||
|
|
debugf("Ignore Mongodb message. Drop tcp stream. Try parsing with the next segment")
|
||
|
|
return conn
|
||
|
|
}
|
||
|
|
|
||
|
|
if !complete {
|
||
|
|
// wait for more data
|
||
|
|
debugf("MongoDB wait for more data before parsing message")
|
||
|
|
break
|
||
|
|
}
|
||
|
|
|
||
|
|
// all ok, go to next level and reset stream for new message
|
||
|
|
debugf("MongoDB message complete")
|
||
|
|
mongodb.handleMongodb(conn, st.message, tcptuple, dir)
|
||
|
|
st.PrepareForNewMessage()
|
||
|
|
}
|
||
|
|
|
||
|
|
return conn
|
||
|
|
}
|
||
|
|
|
||
|
|
func newStream(pkt *protos.Packet, tcptuple *common.TCPTuple) *stream {
|
||
|
|
s := &stream{
|
||
|
|
tcptuple: tcptuple,
|
||
|
|
data: pkt.Payload,
|
||
|
|
message: &mongodbMessage{ts: pkt.Ts},
|
||
|
|
}
|
||
|
|
return s
|
||
|
|
}
|
||
|
|
|
||
|
|
func (mongodb *mongodbPlugin) handleMongodb(
|
||
|
|
conn *mongodbConnectionData,
|
||
|
|
m *mongodbMessage,
|
||
|
|
tcptuple *common.TCPTuple,
|
||
|
|
dir uint8,
|
||
|
|
) {
|
||
|
|
|
||
|
|
m.tcpTuple = *tcptuple
|
||
|
|
m.direction = dir
|
||
|
|
m.cmdlineTuple = procs.ProcWatcher.FindProcessesTupleTCP(tcptuple.IPPort())
|
||
|
|
|
||
|
|
if m.isResponse {
|
||
|
|
debugf("MongoDB response message")
|
||
|
|
mongodb.onResponse(conn, m)
|
||
|
|
} else {
|
||
|
|
debugf("MongoDB request message")
|
||
|
|
mongodb.onRequest(conn, m)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (mongodb *mongodbPlugin) onRequest(conn *mongodbConnectionData, msg *mongodbMessage) {
|
||
|
|
// publish request only transaction
|
||
|
|
if !awaitsReply(msg.opCode) {
|
||
|
|
mongodb.onTransComplete(msg, nil)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
id := msg.requestID
|
||
|
|
key := transactionKey{tcp: msg.tcpTuple.Hashable(), id: id}
|
||
|
|
|
||
|
|
// try to find matching response potentially inserted before
|
||
|
|
if v := mongodb.responses.Delete(key); v != nil {
|
||
|
|
resp := v.(*mongodbMessage)
|
||
|
|
mongodb.onTransComplete(msg, resp)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// insert into cache for correlation
|
||
|
|
old := mongodb.requests.Put(key, msg)
|
||
|
|
if old != nil {
|
||
|
|
debugf("Two requests without a Response. Dropping old request")
|
||
|
|
unmatchedRequests.Add(1)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (mongodb *mongodbPlugin) onResponse(conn *mongodbConnectionData, msg *mongodbMessage) {
|
||
|
|
id := msg.responseTo
|
||
|
|
key := transactionKey{tcp: msg.tcpTuple.Hashable(), id: id}
|
||
|
|
|
||
|
|
// try to find matching request
|
||
|
|
if v := mongodb.requests.Delete(key); v != nil {
|
||
|
|
requ := v.(*mongodbMessage)
|
||
|
|
mongodb.onTransComplete(requ, msg)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// insert into cache for correlation
|
||
|
|
mongodb.responses.Put(key, msg)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (mongodb *mongodbPlugin) onTransComplete(requ, resp *mongodbMessage) {
|
||
|
|
trans := newTransaction(requ, resp)
|
||
|
|
debugf("Mongodb transaction completed: %s", trans.mongodb)
|
||
|
|
|
||
|
|
mongodb.publishTransaction(trans)
|
||
|
|
}
|
||
|
|
|
||
|
|
func newTransaction(requ, resp *mongodbMessage) *transaction {
|
||
|
|
trans := &transaction{}
|
||
|
|
|
||
|
|
// fill request
|
||
|
|
if requ != nil {
|
||
|
|
trans.mongodb = common.MapStr{}
|
||
|
|
trans.event = requ.event
|
||
|
|
trans.method = requ.method
|
||
|
|
|
||
|
|
trans.cmdline = requ.cmdlineTuple
|
||
|
|
trans.ts = requ.ts
|
||
|
|
trans.src, trans.dst = common.MakeEndpointPair(requ.tcpTuple.BaseTuple, requ.cmdlineTuple)
|
||
|
|
if requ.direction == tcp.TCPDirectionReverse {
|
||
|
|
trans.src, trans.dst = trans.dst, trans.src
|
||
|
|
}
|
||
|
|
trans.params = requ.params
|
||
|
|
trans.resource = requ.resource
|
||
|
|
trans.bytesIn = requ.messageLength
|
||
|
|
}
|
||
|
|
|
||
|
|
// fill response
|
||
|
|
if resp != nil {
|
||
|
|
for k, v := range resp.event {
|
||
|
|
trans.event[k] = v
|
||
|
|
}
|
||
|
|
|
||
|
|
trans.error = resp.error
|
||
|
|
trans.documents = resp.documents
|
||
|
|
|
||
|
|
trans.responseTime = int32(resp.ts.Sub(trans.ts).Nanoseconds() / 1e6) // resp_time in milliseconds
|
||
|
|
trans.bytesOut = resp.messageLength
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
return trans
|
||
|
|
}
|
||
|
|
|
||
|
|
func (mongodb *mongodbPlugin) GapInStream(tcptuple *common.TCPTuple, dir uint8,
|
||
|
|
nbytes int, private protos.ProtocolData) (priv protos.ProtocolData, drop bool) {
|
||
|
|
return private, true
|
||
|
|
}
|
||
|
|
|
||
|
|
func (mongodb *mongodbPlugin) ReceivedFin(tcptuple *common.TCPTuple, dir uint8,
|
||
|
|
private protos.ProtocolData) protos.ProtocolData {
|
||
|
|
return private
|
||
|
|
}
|
||
|
|
|
||
|
|
func copyMapWithoutKey(d map[string]interface{}, key string) map[string]interface{} {
|
||
|
|
res := map[string]interface{}{}
|
||
|
|
for k, v := range d {
|
||
|
|
if k != key {
|
||
|
|
res[k] = v
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return res
|
||
|
|
}
|
||
|
|
|
||
|
|
func reconstructQuery(t *transaction, full bool) (query string) {
|
||
|
|
query = t.resource + "." + t.method + "("
|
||
|
|
if len(t.params) > 0 {
|
||
|
|
var err error
|
||
|
|
var params string
|
||
|
|
if !full {
|
||
|
|
// remove the actual data.
|
||
|
|
// TODO: review if we need to add other commands here
|
||
|
|
if t.method == "insert" {
|
||
|
|
params, err = doc2str(copyMapWithoutKey(t.params, "documents"))
|
||
|
|
} else if t.method == "update" {
|
||
|
|
params, err = doc2str(copyMapWithoutKey(t.params, "updates"))
|
||
|
|
} else if t.method == "findandmodify" {
|
||
|
|
params, err = doc2str(copyMapWithoutKey(t.params, "update"))
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
params, err = doc2str(t.params)
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
debugf("Error marshaling params: %v", err)
|
||
|
|
} else {
|
||
|
|
query += params
|
||
|
|
}
|
||
|
|
}
|
||
|
|
query += ")"
|
||
|
|
skip, _ := t.event["numberToSkip"].(int)
|
||
|
|
if skip > 0 {
|
||
|
|
query += fmt.Sprintf(".skip(%d)", skip)
|
||
|
|
}
|
||
|
|
|
||
|
|
limit, _ := t.event["numberToReturn"].(int)
|
||
|
|
if limit > 0 && limit < 0x7fffffff {
|
||
|
|
query += fmt.Sprintf(".limit(%d)", limit)
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
func (mongodb *mongodbPlugin) publishTransaction(t *transaction) {
|
||
|
|
if mongodb.results == nil {
|
||
|
|
debugf("Try to publish transaction with null results")
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
timestamp := t.ts
|
||
|
|
fields := common.MapStr{}
|
||
|
|
fields["type"] = "mongodb"
|
||
|
|
if t.error == "" {
|
||
|
|
fields["status"] = common.OK_STATUS
|
||
|
|
} else {
|
||
|
|
t.event["error"] = t.error
|
||
|
|
fields["status"] = common.ERROR_STATUS
|
||
|
|
}
|
||
|
|
fields["mongodb"] = t.event
|
||
|
|
fields["method"] = t.method
|
||
|
|
fields["resource"] = t.resource
|
||
|
|
fields["query"] = reconstructQuery(t, false)
|
||
|
|
fields["responsetime"] = t.responseTime
|
||
|
|
fields["bytes_in"] = uint64(t.bytesIn)
|
||
|
|
fields["bytes_out"] = uint64(t.bytesOut)
|
||
|
|
fields["src"] = &t.src
|
||
|
|
fields["dst"] = &t.dst
|
||
|
|
|
||
|
|
if mongodb.sendRequest {
|
||
|
|
fields["request"] = reconstructQuery(t, true)
|
||
|
|
}
|
||
|
|
if mongodb.sendResponse {
|
||
|
|
if len(t.documents) > 0 {
|
||
|
|
// response field needs to be a string
|
||
|
|
docs := make([]string, 0, len(t.documents))
|
||
|
|
for i, doc := range t.documents {
|
||
|
|
if mongodb.maxDocs > 0 && i >= mongodb.maxDocs {
|
||
|
|
docs = append(docs, "[...]")
|
||
|
|
break
|
||
|
|
}
|
||
|
|
str, err := doc2str(doc)
|
||
|
|
if err != nil {
|
||
|
|
logp.Warn("Failed to JSON marshal document from Mongo: %v (error: %v)", doc, err)
|
||
|
|
} else {
|
||
|
|
if mongodb.maxDocLength > 0 && len(str) > mongodb.maxDocLength {
|
||
|
|
str = str[:mongodb.maxDocLength] + " ..."
|
||
|
|
}
|
||
|
|
docs = append(docs, str)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
fields["response"] = strings.Join(docs, "\n")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
mongodb.results(beat.Event{
|
||
|
|
Timestamp: timestamp,
|
||
|
|
Fields: fields,
|
||
|
|
})
|
||
|
|
}
|