Logstash Installation
The Logstash package shares the same GPG Key as Elasticsearch, and we already installed that public key, so let’s create and edit a new Yum repository file for Logstash:
sudo vi /etc/yum.repos.d/logstash.repo
Add the following repository configuration:
[logstash-1.4]
name=logstash repository for 1.4.x packages
baseurl=http://packages.elasticsearch.org/logstash/1.4/centos
gpgcheck=1
gpgkey=http://packages.elasticsearch.org/GPG-KEY-elasticsearch
enabled=1
Save and exit.
Install Logstash 1.4.2 with this command:
sudo yum -y install logstash-1.4.2
Create a file called 20-broelastic.conf in /etc/logstash/conf.d
Paste in the following:
#Currently parses the following logs;
#communication.log
#conn.log
#conn-summary.log
#dhcp.log
#dns.log
#dpd.log
#files.log
#http.log
#packet_filter.log
#reporter.log
#software.log
#ssl.log
#weird.log
#notice.log
input {
#Production Logs#############################
file {
type => “BRO_httplog”
path => “/var/opt/bro/logs/current/http.log”
}
file {
type => “BRO_dpdlog”
path => “/var/opt/bro/logs/current/dpd.log”
}
file {
type => “BRO_connlog”
path => “/var/opt/bro/logs/current/conn.log”
}
file {
type => “BRO_weirdlog”
path => “/var/opt/bro/logs/current/weird.log”
}
file {
type => “BRO_appstatslog”
path => “/var/opt/bro/logs/current/appstats.log”
}
file {
type => “BRO_communicationlog”
path => “/var/opt/bro/logs/current/communication.log”
}
file {
type => “BRO_dhcplog”
path => “/var/opt/bro/logs/current/dhcp.log”
}
file {
type => “BRO_fileslog”
path => “/var/opt/bro/logs/current/files.log”
}
file {
type => “BRO_SSLlog”
path => “/var/opt/bro/logs/current/ssl.log”
}
file {
type => “BRO_noticelog”
path => “/var/opt/bro/logs/current/notice.log”
}
file {
type => “BRO_softwarelog”
path => “/var/opt/bro/logs/current/software.log”
}
file {
type => “BRO_reporterlog”
path => “/var/opt/bro/logs/current/reporter.log”
}
file {
type => “BRO_packetfilterlog”
path => “/var/opt/bro/logs/current/packet_filter.log”
}
file {
type => “BRO_dnslog”
path => “/var/opt/bro/logs/current/dns.log”
}
#Test Logs#############################################
# file {
# type => “BRO_httplogTEST”
# path => “/home/idsusr/brotest/http.log”
# }
# file {
# type => “BRO_noticelogTEST”
# path => “/home/idsusr/brotest/notice.log”
# }
#######################################################
}
filter {
if [message] =~ /^#/ {
drop { }
} else {
# BRO_httplog ######################
if [type] == “BRO_httplog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<uid>(.*?))t(?<id.orig_h>(.*?))t(?<id.orig_p>(.*?))t(?<id.resp_h>(.*?))t(?<id.resp_p>(.*?))t(?<trans_depth>(.*?))t(?<method>(.*?))t(?<host>(.*?))t(?<uri>(.*?))t(?<referrer>(.*?))t(?<user_agent>(.*?))t(?<request_body_len>(.*?))t(?<response_body_len>(.*?))t(?<status_code>(.*?))t(?<status_msg>(.*?))t(?<info_code>(.*?))t(?<info_msg>(.*?))t(?<filename>(.*?))t(?<tags>(.*?))t(?<username>(.*?))t(?<password>(.*?))t(?<proxied>(.*?))t(?<orig_fuids>(.*?))t(?<orig_mime_types>(.*?))t(?<resp_fuids>(.*?))t(?<resp_mime_types>(.*))” ]
}
}
# BRO_dpdlog ######################
if [type] == “BRO_dpdlog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<uid>(.*?))t(?<id.orig_h>(.*?))t(?<id.orig_p>(.*?))t(?<id.resp_h>(.*?))t(?<id.resp_p>(.*?))t(?<proto>(.*?))t(?<analyzer>(.*?))t(?<failure_reason>(.*))” ]
}
}
# BRO_connlog ######################
if [type] == “BRO_connlog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<uid>(.*?))t(?<id.orig_h>(.*?))t(?<id.orig_p>(.*?))t(?<id.resp_h>(.*?))t(?<id.resp_p>(.*?))t(?<proto>(.*?))t(?<service>(.*?))t(?<duration>(.*?))t(?<orig_bytes>(.*?))t(?<resp_bytes>(.*?))t(?<conn_state>(.*?))t(?<local_orig>(.*?))t(?<missed_bytes>(.*?))t(?<history>(.*?))t(?<orig_pkts>(.*?))t(?<orig_ip_bytes>(.*?))t(?<resp_pkts>(.*?))t(?<resp_ip_bytes>(.*?))t(?<tunnel_parents>(.*))” ]
}
}
# BRO_weirdlog ######################
if [type] == “BRO_weirdlog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<uid>(.*?))t(?<id.orig_h>(.*?))t(?<id.orig_p>(.*?))t(?<id.resp_h>(.*?))t(?<id.resp_p>(.*?))t(?<name>(.*?))t(?<addl>(.*?))t(?<notice>(.*?))t(?<peer>(.*))” ]
}
}
# BRO_appstatslog ######################
if [type] == “BRO_appstatslog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<ts_delta>(.*?))t(?<app>(.*?))t(?<uniq_hosts>(.*?))t(?<hits>(.*?))t(?<bytes>(.*))” ]
}
}
# BRO_communicationlog ######################
if [type] == “BRO_communicationlog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<peer>(.*?))t(?<src_name>(.*?))t(?<connected_peer_desc>(.*?))t(?<connected_peer_addr>(.*?))t(?<connected_peer_port>(.*?))t(?<level>(.*?))t(?<message>(.*))” ]
}
}
# BRO_dhcplog ######################
if [type] == “BRO_dhcplog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<uid>(.*?))t(?<id.orig_h>(.*?))t(?<id.orig_p>(.*?))t(?<id.resp_h>(.*?))t(?<id.resp_p>(.*?))t(?<mac>(.*?))t(?<assigned_ip>(.*?))t(?<lease_time>(.*?))t(?<trans_id>(.*))” ]
}
}
# BRO_fileslog ######################
if [type] == “BRO_fileslog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<fuid>(.*?))t(?<tx_hosts>(.*?))t(?<rx_hosts>(.*?))t(?<conn_uids>(.*?))t(?<source>(.*?))t(?<depth>(.*?))t(?<analyzers>(.*?))t(?<mime_type>(.*?))t(?<filename>(.*?))t(?<duration>(.*?))t(?<local_orig>(.*?))t(?<is_orig>(.*?))t(?<seen_bytes>(.*?))t(?<total_bytes>(.*?))t(?<missing_bytes>(.*?))t(?<overflow_bytes>(.*?))t(?<timedout>(.*?))t(?<parent_fuid>(.*?))t(?<md5>(.*?))t(?<sha1>(.*?))t(?<sha256>(.*?))t(?<extracted>(.*))” ]
}
}
# BRO_SSLlog ######################
if [type] == “BRO_SSLlog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<uid>(.*?))t(?<id.orig_h>(.*?))t(?<id.orig_p>(.*?))t(?<id.resp_h>(.*?))t(?<id.resp_p>(.*?))t(?<version>(.*?))t(?<cipher>(.*?))t(?<server_name>(.*?))t(?<session_id>(.*?))t(?<subject>(.*?))t(?<issuer_subject>(.*?))t(?<not_valid_before>(.*?))t(?<not_valid_after>(.*?))t(?<last_alert>(.*?))t(?<client_subject>(.*?))t(?<client_issuer_subject>(.*?))t(?<cert_hash>(.*?))t(?<validation_status>(.*))” ]
}
}
# BRO_noticelog ######################
if [type] == “BRO_noticelog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<uid>(.*?))t(?<id.orig_h>(.*?))t(?<id.orig_p>(.*?))t(?<id.resp_h>(.*?))t(?<id.resp_p>(.*?))t(?<fuid>(.*?))t(?<file_mime_type>(.*?))t(?<file_desc>(.*?))t(?<proto>(.*?))t(?<note>(.*?))t(?<msg>(.*?))t(?<sub>(.*?))t(?<src>(.*?))t(?<dst>(.*?))t(?<p>(.*?))t(?<n>(.*?))t(?<peer_descr>(.*?))t(?<actions>(.*?))t(?<suppress_for>(.*?))t(?<dropped>(.*?))t(?<remote_location.country_code>(.*?))t(?<remote_location.region>(.*?))t(?<remote_location.city>(.*?))t(?<remote_location.latitude>(.*?))t(?<remote_location.longitude>(.*))” ]
}
}
# BRO_softwarelog ######################
if [type] == “BRO_softwarelog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<host>(.*?))t(?<host_p>(.*?))t(?<software_type>(.*?))t(?<name>(.*?))t(?<version.major>(.*?))t(?<version.minor>(.*?))t(?<version.minor2>(.*?))t(?<version.minor3>(.*?))t(?<version.addl>(.*?))t(?<unparsed_version>(.*))” ]
}
}
# BRO_reporterlog ######################
if [type] == “BRO_noticelog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<level>(.*?))t(?<message>(.*?))t(?<location>(.*))” ]
}
}
# BRO_packetfilterlog ######################
if [type] == “BRO_packetfilterlog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<node>(.*?))t(?<filter>(.*?))t(?<init>(.*?))t(?<success>(.*))” ]
}
}
# BRO_dnslog ######################
if [type] == “BRO_dnslog” {
grok {
match => [ “message”, “(?<ts>(.*?))t(?<uid>(.*?))t(?<id.orig_h>(.*?))t(?<id.orig_p>(.*?))t(?<id.resp_h>(.*?))t(?<id.resp_p>(.*?))t(?<proto>(.*?))t(?<trans_id>(.*?))t(?<query>(.*?))t(?<qclass>(.*?))t(?<qclass_name>(.*?))t(?<qtype>(.*?))t(?<qtype_name>(.*?))t(?<rcode>(.*?))t(?<rcode_name>(.*?))t(?<AA>(.*?))t(?<TC>(.*?))t(?<RD>(.*?))t(?<RA>(.*?))t(?<Z>(.*?))t(?<answers>(.*?))t(?<TTLs>(.*?))t(?<rejected>(.*))” ]
}
}
}
}
output {
elasticsearch { host => localhost }
}
Restart logstash
service logstash restart
0 Comments