// Licensed to Elasticsearch B.V. under one or more contributor // license agreements. See the NOTICE file distributed with // this work for additional information regarding copyright // ownership. Elasticsearch B.V. licenses this file to you under // the Apache License, Version 2.0 (the "License"); you may // not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. package pgsql import ( "errors" "strings" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" ) var ( errInvalidString = errors.New("invalid pgsql string") errEmptyFieldsBuffer = errors.New("empty fields buffer") errNoFieldName = errors.New("can not read column field") errFieldBufferShort = errors.New("field buffer to small for field count") errFieldBufferBig = errors.New("field count to small for field buffer size") ) func (pgsql *pgsqlPlugin) pgsqlMessageParser(s *pgsqlStream) (bool, bool) { debugf("pgsqlMessageParser, off=%v", s.parseOffset) var ok, complete bool switch s.parseState { case pgsqlStartState: ok, complete = pgsql.parseMessageStart(s) case pgsqlGetDataState: ok, complete = pgsql.parseMessageData(s) case pgsqlExtendedQueryState: ok, complete = pgsql.parseMessageExtendedQuery(s) default: logp.Critical("Pgsql invalid parser state") } detailedf("pgsqlMessageParser return: ok=%v, complete=%v, off=%v", ok, complete, s.parseOffset) return ok, complete } func (pgsql *pgsqlPlugin) parseMessageStart(s *pgsqlStream) (bool, bool) { detailedf("parseMessageStart") m := s.message for len(s.data[s.parseOffset:]) >= 5 { isSpecial, length, command := isSpecialPgsqlCommand(s.data[s.parseOffset:]) if !isSpecial { return pgsql.parseCommand(s) } // In case of Commands: StartupMessage, SSLRequest, CancelRequest that don't have // their type in the first byte // check buffer available if len(s.data[s.parseOffset:]) <= length { detailedf("Wait for more data 1") return true, false } // ignore non SSLRequest commands if command != sslRequest { s.parseOffset += length continue } // if SSLRequest is received, expect for one byte reply (S or N) m.start = s.parseOffset s.parseOffset += length m.end = s.parseOffset m.isSSLRequest = true m.size = uint64(m.end - m.start) return true, true } return true, false } func (pgsql *pgsqlPlugin) parseCommand(s *pgsqlStream) (bool, bool) { // read type typ := byte(s.data[s.parseOffset]) if s.expectSSLResponse { // SSLRequest was received in the other stream if typ == 'N' || typ == 'S' { m := s.message // one byte reply to SSLRequest detailedf("Reply for SSLRequest %c", typ) m.start = s.parseOffset s.parseOffset++ m.end = s.parseOffset m.isSSLResponse = true m.size = uint64(m.end - m.start) return true, true } } // read length length := readLength(s.data[s.parseOffset+1:]) if length < 4 { // length should include the size of itself (int32) detailedf("Invalid pgsql command length.") return false, false } if len(s.data[s.parseOffset:]) <= length { detailedf("Wait for more data") return true, false } detailedf("Pgsql type %c, length=%d", typ, length) switch typ { case 'Q': return pgsql.parseSimpleQuery(s, length) case 'T': return pgsql.parseRowDescription(s, length) case 'I': return pgsql.parseEmptyQueryResponse(s) case 'C': return pgsql.parseCommandComplete(s, length) case 'Z': return pgsql.parseReadyForQuery(s, length) case 'E': return pgsql.parseErrorResponse(s, length) case 'P': return pgsql.parseExtReq(s, length) case '1': return pgsql.parseExtResp(s, length) default: if !pgsqlValidType(typ) { detailedf("invalid frame type: '%c'", typ) return false, false } return pgsql.parseSkipMessage(s, length) } } func (pgsql *pgsqlPlugin) parseSimpleQuery(s *pgsqlStream, length int) (bool, bool) { m := s.message m.start = s.parseOffset m.isRequest = true s.parseOffset++ //type s.parseOffset += length m.end = s.parseOffset m.size = uint64(m.end - m.start) query, err := pgsqlString(s.data[m.start+5:], length-4) if err != nil { return false, false } m.query = query m.toExport = true detailedf("Simple Query: %s", m.query) return true, true } func (pgsql *pgsqlPlugin) parseRowDescription(s *pgsqlStream, length int) (bool, bool) { // RowDescription m := s.message m.start = s.parseOffset m.isRequest = false m.isOK = true m.toExport = true err := pgsqlFieldsParser(s, s.data[s.parseOffset+5:s.parseOffset+length+1]) if err != nil { detailedf("fields parse failed with: %v", err) return false, false } detailedf("Fields: %s", m.fields) s.parseOffset++ //type s.parseOffset += length //length s.parseState = pgsqlGetDataState return pgsql.parseMessageData(s) } // Parse a list of commands separated by semicolon from the query func pgsqlQueryParser(query string) []string { array := strings.Split(query, ";") queries := []string{} for _, q := range array { qt := strings.TrimSpace(q) if len(qt) > 0 { queries = append(queries, qt) } } return queries } func (pgsql *pgsqlPlugin) parseEmptyQueryResponse(s *pgsqlStream) (bool, bool) { // EmptyQueryResponse, appears as a response for empty queries // substitutes CommandComplete m := s.message detailedf("EmptyQueryResponse") m.start = s.parseOffset m.isOK = true m.isRequest = false m.toExport = true s.parseOffset += 5 // type + length m.end = s.parseOffset m.size = uint64(m.end - m.start) return true, true } func (pgsql *pgsqlPlugin) parseCommandComplete(s *pgsqlStream, length int) (bool, bool) { // CommandComplete -> Successful response m := s.message m.start = s.parseOffset m.isRequest = false m.isOK = true m.toExport = true s.parseOffset++ //type name, err := pgsqlString(s.data[s.parseOffset+4:], length-4) if err != nil { return false, false } detailedf("CommandComplete length=%d, tag=%s", length, name) s.parseOffset += length m.end = s.parseOffset m.size = uint64(m.end - m.start) return true, true } func (pgsql *pgsqlPlugin) parseReadyForQuery(s *pgsqlStream, length int) (bool, bool) { // ReadyForQuery -> backend ready for a new query cycle m := s.message m.start = s.parseOffset m.size = uint64(m.end - m.start) s.parseOffset++ // type s.parseOffset += length m.end = s.parseOffset return true, true } func (pgsql *pgsqlPlugin) parseErrorResponse(s *pgsqlStream, length int) (bool, bool) { // ErrorResponse detailedf("ErrorResponse") m := s.message m.start = s.parseOffset m.isRequest = false m.isError = true m.toExport = true s.parseOffset++ //type pgsqlErrorParser(s, s.data[s.parseOffset+4:s.parseOffset+length]) s.parseOffset += length //length m.end = s.parseOffset m.size = uint64(m.end - m.start) return true, true } func (pgsql *pgsqlPlugin) parseExtReq(s *pgsqlStream, length int) (bool, bool) { // Ready for query -> Parse for an extended query request detailedf("Parse") m := s.message m.start = s.parseOffset m.isRequest = true s.parseOffset++ //type s.parseOffset += length m.end = s.parseOffset m.size = uint64(m.end - m.start) m.toExport = true query, err := common.ReadString(s.data[m.start+6:]) if err != nil { detailedf("Invalid extended query request") return false, false } m.query = query detailedf("Parse in an extended query request: %s", m.query) // Ignore SET statement if strings.HasPrefix(m.query, "SET ") { m.toExport = false } s.parseState = pgsqlExtendedQueryState return pgsql.parseMessageExtendedQuery(s) } func (pgsql *pgsqlPlugin) parseExtResp(s *pgsqlStream, length int) (bool, bool) { // Sync -> Parse completion for an extended query response detailedf("ParseCompletion") m := s.message m.start = s.parseOffset m.isRequest = false m.isOK = true m.toExport = true s.parseOffset++ //type s.parseOffset += length detailedf("Parse completion in an extended query response") s.parseState = pgsqlGetDataState return pgsql.parseMessageData(s) } func (pgsql *pgsqlPlugin) parseSkipMessage(s *pgsqlStream, length int) (bool, bool) { // TODO: add info from NoticeResponse in case there are warning messages for a query // ignore command s.parseOffset++ //type s.parseOffset += length m := s.message m.end = s.parseOffset m.size = uint64(m.end - m.start) // ok and complete, but ignore m.toExport = false return true, true } func pgsqlFieldsParser(s *pgsqlStream, buf []byte) error { m := s.message if len(buf) < 2 { return errEmptyFieldsBuffer } // read field count (int16) off := 2 fieldCount := readCount(buf) detailedf("Row Description field count=%d", fieldCount) fields := []string{} fieldsFormat := []byte{} for i := 0; i < fieldCount; i++ { if len(buf) <= off { return errFieldBufferShort } // read field name (null terminated string) fieldName, err := common.ReadString(buf[off:]) if err != nil { return errNoFieldName } fields = append(fields, fieldName) m.numberOfFields++ off += len(fieldName) + 1 // read Table OID (int32) off += 4 // read Column Index (int16) off += 2 // read Type OID (int32) off += 4 // read column length (int16) off += 2 // read type modifier (int32) off += 4 // read format (int16) if len(buf) < off+2 { return errFieldBufferShort } format := common.BytesNtohs(buf[off : off+2]) off += 2 fieldsFormat = append(fieldsFormat, byte(format)) detailedf("Field name=%s, format=%d", fieldName, format) } if off < len(buf) { return errFieldBufferBig } m.fields = fields m.fieldsFormat = fieldsFormat if m.numberOfFields != fieldCount { logp.Err("Missing fields from RowDescription. Expected %d. Received %d", fieldCount, m.numberOfFields) } return nil } func pgsqlErrorParser(s *pgsqlStream, buf []byte) { m := s.message off := 0 for off < len(buf) { // read field type(byte1) typ := buf[off] if typ == 0 { break } // read field value(string) val, err := common.ReadString(buf[off+1:]) if err != nil { logp.Err("Failed to read the column field") break } off += len(val) + 2 switch typ { case 'M': m.errorInfo = val case 'C': m.errorCode = val case 'S': m.errorSeverity = val } } detailedf("%s %s %s", m.errorSeverity, m.errorCode, m.errorInfo) } func (pgsql *pgsqlPlugin) parseMessageData(s *pgsqlStream) (bool, bool) { detailedf("parseMessageData") // The response to queries that return row sets contains: // RowDescription // zero or more DataRow // CommandComplete // ReadyForQuery m := s.message for len(s.data[s.parseOffset:]) > 5 { // read type typ := byte(s.data[s.parseOffset]) // read message length length := readLength(s.data[s.parseOffset+1:]) if length < 4 { // length should include the size of itself (int32) detailedf("Invalid pgsql command length.") return false, false } if len(s.data[s.parseOffset:]) <= length { // wait for more detailedf("Wait for more data") return true, false } switch typ { case 'D': err := pgsql.parseDataRow(s, s.data[s.parseOffset+5:s.parseOffset+length+1]) if err != nil { return false, false } s.parseOffset++ s.parseOffset += length case 'C': // CommandComplete // skip type s.parseOffset++ name, err := pgsqlString(s.data[s.parseOffset+4:], length-4) if err != nil { detailedf("pgsql string invalid") return false, false } detailedf("CommandComplete length=%d, tag=%s", length, name) s.parseOffset += length m.end = s.parseOffset m.size = uint64(m.end - m.start) s.parseState = pgsqlStartState detailedf("Rows: %s", m.rows) return true, true case '2': // Parse completion -> Bind completion for an extended query response // skip type s.parseOffset++ s.parseOffset += length s.parseState = pgsqlStartState case 'T': return pgsql.parseRowDescription(s, length) default: // shouldn't happen -> return error logp.Warn("Pgsql parser expected data message, but received command of type %v", typ) s.parseState = pgsqlStartState return false, false } } return true, false } func (pgsql *pgsqlPlugin) parseDataRow(s *pgsqlStream, buf []byte) error { m := s.message // read field count (int16) off := 2 fieldCount := readCount(buf) detailedf("DataRow field count=%d", fieldCount) rows := []string{} rowLength := 0 for i := 0; i < fieldCount; i++ { if len(buf) <= off { return errFieldBufferShort } // read column length (int32) columnLength := readLength(buf[off:]) off += 4 if columnLength > 0 && columnLength > len(buf[off:]) { logp.Err("Pgsql invalid column_length=%v, buffer_length=%v, i=%v", columnLength, len(buf[off:]), i) return errInvalidLength } // read column value (byten) var columnValue []byte if m.fieldsFormat[i] == 0 { // field value in text format if columnLength > 0 { columnValue = buf[off : off+columnLength] off += columnLength } } if rowLength < pgsql.maxRowLength { if rowLength+len(columnValue) > pgsql.maxRowLength { columnValue = columnValue[:pgsql.maxRowLength-rowLength] } rows = append(rows, string(columnValue)) rowLength += len(columnValue) } detailedf("Value %s, length=%d, off=%d", string(columnValue), columnLength, off) } if off < len(buf) { return errFieldBufferBig } m.numberOfRows++ if len(m.rows) < pgsql.maxStoreRows { m.rows = append(m.rows, rows) } return nil } func (pgsql *pgsqlPlugin) parseMessageExtendedQuery(s *pgsqlStream) (bool, bool) { detailedf("parseMessageExtendedQuery") // An extended query request contains: // Parse // Bind // Describe // Execute // Sync m := s.message for len(s.data[s.parseOffset:]) >= 5 { // read type typ := byte(s.data[s.parseOffset]) // read message length length := readLength(s.data[s.parseOffset+1:]) if length < 4 { // length should include the size of itself (int32) detailedf("Invalid pgsql command length.") return false, false } if len(s.data[s.parseOffset:]) <= length { // wait for more detailedf("Wait for more data") return true, false } switch typ { case 'B': // Parse -> Bind // skip type s.parseOffset++ s.parseOffset += length //TODO: pgsql.parseBind(s) case 'D': // Bind -> Describe // skip type s.parseOffset++ s.parseOffset += length //TODO: pgsql.parseDescribe(s) case 'E': // Bind(or Describe) -> Execute // skip type s.parseOffset++ s.parseOffset += length //TODO: pgsql.parseExecute(s) case 'S': // Execute -> Sync // skip type s.parseOffset++ s.parseOffset += length m.end = s.parseOffset m.size = uint64(m.end - m.start) s.parseState = pgsqlStartState return true, true default: // shouldn't happen -> return error logp.Warn("Pgsql parser expected extended query message, but received command of type %v", typ) s.parseState = pgsqlStartState return false, false } } return true, false } func isSpecialPgsqlCommand(data []byte) (bool, int, int) { if len(data) < 8 { // 8 bytes required return false, 0, 0 } // read length length := readLength(data[0:]) // read command identifier code := int(common.BytesNtohl(data[4:])) if length == 16 && code == 80877102 { // Cancel Request logp.Debug("pgsqldetailed", "Cancel Request, length=%d", length) return true, length, cancelRequest } else if length == 8 && code == 80877103 { // SSL Request logp.Debug("pgsqldetailed", "SSL Request, length=%d", length) return true, length, sslRequest } else if code == 196608 { // Startup Message logp.Debug("pgsqldetailed", "Startup Message, length=%d", length) return true, length, startupMessage } return false, 0, 0 } // length field in pgsql counts total length of length field + payload, not // including the message identifier. => Always check buffer size >= length + 1 func readLength(b []byte) int { return int(common.BytesNtohl(b)) } func readCount(b []byte) int { return int(common.BytesNtohs(b)) } func pgsqlString(b []byte, sz int) (string, error) { if sz == 0 { return "", nil } if b[sz-1] != 0 { return "", errInvalidString } return string(b[:sz-1]), nil } func pgsqlValidType(t byte) bool { switch t { case '1', '2', '3', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'K', 'N', 'P', 'Q', 'R', 'S', 'T', 'V', 'W', 'X', 'Z', 'c', 'd', 'f', 'n', 'p', 's', 't': return true default: return false } }