Logging using ELK Stack

# CustomLog ${APACHE_LOG_DIR}/access_log combined
        CustomLog ${APACHE_LOG_DIR}/access_log tikicombined
        LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" \"%{X-Remote-User}o\" \"%{X-Current-Object}o\" \"%{X-Tiki-Events}o\" %{X-Tiki-Id}o log_yourindexname" tikicombined

# Turning on user and object information to be passed to server logs
SetEnv TIKI_HEADER_REPORT_ID my_tiki
SetEnv TIKI_HEADER_REPORT_USER on
SetEnv TIKI_HEADER_REPORT_OBJECT on
SetEnv TIKI_HEADER_REPORT_EVENTS on

filebeat:
  prospectors:
    -
      paths:
        - /var/log/apache2/access_log

      input_type: log

      document_type: tikicombined

output:
  logstash:
    hosts: ["logstash.yourdomain.com:5043"]

    # SSL settings to connect to the logstash server if needed
    tls:
      # List of root certificates for HTTPS server verifications
      certificate_authorities: ["/etc/filebeat/logstash_ssl_cert.pem"]
      # Note this is the full concatenated cert+chain that is for the logstash server

shipper:
  # The name of the shipper that publishes the network data. It can be used to group
  # all the transactions sent by a single shipper in the web interface.
  # If this options is not defined, the hostname is used.
  name: your_site_name

input {
    beats {
        port => "5043"
        ssl => true
        # set to false if ssl if not needed
        ssl_certificate => "logstash_ssl_cert.pem" 
        ssl_key => "/etc/logstash/logstash_private_key.pem" 
        # note that this key should be kept only on this server and be readable only by the logstash daemon and root.
    }
}

filter {
    # read supported document types set in filebeat
    if [type] == "tikicombined" { 
        grok {
            match => { "message" => "%{COMBINEDAPACHELOG} %{DATA:remoteuser} %{QS:currentobject} %{QS:events} %{WORD:tiki_instance} %{WORD:[@metadata][index]}" }
        }
    } else {
        # unsupported document type
        mutate { add_tag => "drop" }
    }

    # flag error logs to be stored in different index
    if [response] =~ /^5\d\d/ {
        mutate { add_tag => [ "error", "5xx" ] }
    }
    if [response] =~ /^4\d\d/ {
        mutate { add_tag => [ "error", "4xx" ] }
    }

    # index needs to be specified to work
    if ![@metadata][index] {
        mutate { add_tag => "drop" }
    } 

    # get log filename without path 
    grok {
        match => { "source" => "%{GREEDYDATA}/%{GREEDYDATA:logfilename}" }
        break_on_match => false
    }

    # get query string parameters
    grok {
        match => { "request" => "%{URIPARAM:querystring}" }
        # do not add _grokparsefailure tag if query string is not found
        tag_on_failure => []
    }
    mutate {
        gsub => [
            "querystring", "\?", ""
        ]
    }
    kv {
        source => 'querystring'
        field_split => '&'
    	target => 'queryparams'
    }

    # get referrer query string parameters
    if [referrer] {
        grok {
            match => { "referrer" => "%{URIPARAM:referrer_querystring}" }
            # do not add _grokparsefailure tag if query string is not found
            tag_on_failure => []
        }
        mutate {
            gsub => [
                "referrer_querystring", "\?", ""
            ]
        }
        kv {
            source => 'referrer_querystring'
            field_split => '&'
            target => 'referrer_queryparams'
        }
    }

    # replace dash value with nothing
    mutate {
        gsub => [
            "referrer", "\"-\"", "",
            "remoteuser", "\"-\"", "",
            "currentobject", "\"-\"", "",
            "events", "\"-\"", ""
        ]
    }

    # clean up escaped json quotes
    mutate {
        gsub => [
            "events", "\\\"", '"'
        ]
    }

    # clean up double quotes
    mutate {
        gsub => [
            "referrer", "^\"", "",
            "remoteuser", "^\"", "",
            "currentobject", "^\"", "",
            "events", "^\"", "" 
        ]
    }
    mutate {
        gsub => [
            "referrer", "\"$", "",
            "remoteuser", "\"$", "",
            "currentobject", "\"$", "",
            "events", "\"$", ""
        ]
    }

    # throttle repeated events if desired (performance warning: disables multithreaded pipelining)
#    throttle {
#        after_count => 1
#        period => 600
#        key => "%{agent}%{clientip}%{request}%{remoteuser}%{currentobject}"
#        add_tag => "throttled"
#    }

    if "throttled" in [tags] {
        mutate { add_tag => "drop" }
    } else if [tiki_instance] == "" {
	# drop non Tiki generated logs
	mutate { add_tag => "drop" }
    } else {
        # get Tiki object information
        mutate {
            split => { "currentobject" => ":" }
        }
        if [currentobject][0] and [currentobject][1] {
            mutate {
                add_field => { "object_type" => "%{[currentobject][0]}" }
                add_field => { "object_id" => "%{[currentobject][1]}" }
            }
        } else {
            mutate {
                add_field => { "object_type" => "" }
                add_field => { "object_id" => "" }
            }
        }

        # replace anonymous username
        if [remoteuser] == "" {
            mutate {
                replace => { "remoteuser" => "-anonymous-" }
            }
        }

        # convert events to json
        json {
            source => events
            target => events
        }

        # get user location info from ip
        geoip {
            source => "clientip"
            target => "geoip"
        }

        # parse useragent info
        useragent {
            source => "agent"
            target => "useragent"
        }
    }

    # drop unneeded
    if "drop" in [tags] {
        drop { }
    }
}

output {
    # if it is an error, log to a different index. (if desired)
    if "error" in [tags] {
        elasticsearch {
           #this is an example of authenticated access at Elastic Cloud. Modify as necessary
            hosts => ["https://whatever.us-east-1.aws.found.io:9243"]
            user => "logstash"
            password => "whatever"
            index => "%{[@metadata][index]}_errors"
            manage_template => false 
        }
    } else {
        elasticsearch {
           #this is an example of authenticated access at Elastic Cloud. Modify as necessary
            hosts => ["https://whatever.us-east-1.aws.found.io:9243"]
            user => "logstash"
            password => "whatever"
            index => "%{[@metadata][index]}" 
            manage_template => false
        }
    }
#    stdout {
#        codec => rubydebug {
#            metadata => true
#        }
#    }
# uncomment the stdout for debugging purposes. If logstash is run from the console you should see the output, if run as a service it should log somewhere like /var/log/logstash/logstash.stdout (or similar).
}