606 lines
16 KiB
JSON
606 lines
16 KiB
JSON
{
|
|
"filter_ogdhcp_pipeline" : {
|
|
"description" : "Parse logs to extract http_code and desc, while preserving original message",
|
|
"processors" : [
|
|
{
|
|
"script" : {
|
|
"if" : "ctx.syslog?.identifier != 'ogdhcp'",
|
|
"source" : "\n ctx.debug = 'Skipped: identifier is ' + (ctx.syslog?.identifier ?: 'undefined');\n ctx.pipeline_stop = true; // Stops further processing but retains the document\n "
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "debug",
|
|
"value" : "Processed: identifier is ogdhcp"
|
|
}
|
|
},
|
|
{
|
|
"script" : {
|
|
"source" : "\n ctx.processed_message = ctx.message;\n "
|
|
}
|
|
},
|
|
{
|
|
"gsub" : {
|
|
"field" : "processed_message",
|
|
"pattern" : "^app\\.[A-Z]+: ",
|
|
"replacement" : "",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"gsub" : {
|
|
"field" : "processed_message",
|
|
"pattern" : "^request\\.INFO: Matched route \".*?\"\\. ",
|
|
"replacement" : "",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"json" : {
|
|
"field" : "processed_message",
|
|
"target_field" : "parsed_message",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "route",
|
|
"value" : "{{parsed_message.route}}",
|
|
"ignore_empty_value" : true,
|
|
"if" : "ctx.parsed_message?.route != null"
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "route_parameters",
|
|
"value" : "{{parsed_message.route_parameters}}",
|
|
"ignore_empty_value" : true,
|
|
"if" : "ctx.parsed_message?.route_parameters != null"
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "request_uri",
|
|
"value" : "{{parsed_message.request_uri}}",
|
|
"ignore_empty_value" : true,
|
|
"if" : "ctx.parsed_message?.request_uri != null"
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "method",
|
|
"value" : "{{parsed_message.method}}",
|
|
"ignore_empty_value" : true,
|
|
"if" : "ctx.parsed_message?.method != null"
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "http_code",
|
|
"value" : "{{parsed_message.http_code}}",
|
|
"ignore_empty_value" : true
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "description",
|
|
"value" : "{{parsed_message.desc}}",
|
|
"ignore_empty_value" : true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"master_pipeline" : {
|
|
"description" : "Master pipeline to route logs based on syslog.identifier",
|
|
"processors" : [
|
|
{
|
|
"pipeline" : {
|
|
"name" : "filter_tftp_pipeline",
|
|
"if" : "ctx.syslog?.identifier == 'in.tftpd'"
|
|
}
|
|
},
|
|
{
|
|
"pipeline" : {
|
|
"name" : "filter_ogboot_pipeline",
|
|
"if" : "ctx.syslog?.identifier == 'ogboot'"
|
|
}
|
|
},
|
|
{
|
|
"pipeline" : {
|
|
"name" : "filter_ogdhcp_pipeline",
|
|
"if" : "ctx.syslog?.identifier == 'ogdhcp'"
|
|
}
|
|
},
|
|
{
|
|
"pipeline" : {
|
|
"name" : "kea_dhcp_pipeline",
|
|
"if" : "ctx.syslog?.identifier == 'kea-dhcp4'"
|
|
}
|
|
},
|
|
{
|
|
"pipeline" : {
|
|
"name" : "ogrepo_pipeline",
|
|
"if" : "ctx.syslog?.identifier == 'ogrepo-api'"
|
|
}
|
|
},
|
|
{
|
|
"pipeline" : {
|
|
"name" : "docker_logs_pipeline",
|
|
"if" : "ctx.syslog?.identifier == 'docker'"
|
|
}
|
|
},
|
|
{
|
|
"json": {
|
|
"field": "message",
|
|
"target_field": "parsed_message",
|
|
"ignore_failure": true,
|
|
"if": "ctx.syslog?.identifier == 'ogcore'"
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "debug",
|
|
"value" : "No matching pipeline, skipping further processing.",
|
|
"if" : "ctx.syslog?.identifier != 'in.tftpd' && ctx.syslog?.identifier != 'ogboot' && ctx.syslog?.identifier != 'kea-dhcp4' && ctx.syslog?.identifier != 'ogrepo-api' && ctx.syslog?.identifier != 'docker'"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"json_parse_pipeline" : {
|
|
"description" : "Parse JSON payload from logs",
|
|
"processors" : [
|
|
{
|
|
"json" : {
|
|
"field" : "message",
|
|
"target_field" : "parsed_json",
|
|
"ignore_failure" : true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"docker_logs_pipeline" : {
|
|
"description" : "Parse Docker logs and route based on container name",
|
|
"processors" : [
|
|
{
|
|
"grok" : {
|
|
"field" : "message",
|
|
"patterns" : [
|
|
"%{DATA:container.name}\\s*\\|%{GREEDYDATA:log_details}"
|
|
],
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"pipeline" : {
|
|
"name" : "parse_nginx_logs",
|
|
"if" : "ctx.container?.name == 'ogcore-nginx'",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"json" : {
|
|
"field" : "log_details",
|
|
"target_field" : "parsed_json",
|
|
"ignore_failure" : true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"json_parse_with_replacement_debug" : {
|
|
"description" : "Debug replacement of single quotes with double quotes and parse JSON",
|
|
"processors" : [
|
|
{
|
|
"script" : {
|
|
"source" : "\n ctx.message = ctx.message.replace(\"'\", \"\\\"\");\n "
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "debug_message",
|
|
"value" : "{{ message }}"
|
|
}
|
|
},
|
|
{
|
|
"json" : {
|
|
"field" : "message",
|
|
"target_field" : "parsed_json",
|
|
"ignore_failure" : true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"ogrepo_parse_pipeline" : {
|
|
"description" : "Parse ogRepo logs for detailed JSON information",
|
|
"processors" : [
|
|
{
|
|
"grok" : {
|
|
"field" : "message",
|
|
"patterns" : [
|
|
"%{TIMESTAMP_ISO8601:timestamp} %{DATA:hostname} %{DATA:service}\\[%{NUMBER:pid}\\]: %{GREEDYDATA:json_payload}"
|
|
],
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"json" : {
|
|
"field" : "json_payload",
|
|
"target_field" : "parsed_json",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"rename" : {
|
|
"field" : "parsed_json.component",
|
|
"target_field" : "component",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"rename" : {
|
|
"field" : "parsed_json.severity",
|
|
"target_field" : "severity",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"rename" : {
|
|
"field" : "parsed_json.http_code",
|
|
"target_field" : "http_code",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"rename" : {
|
|
"field" : "parsed_json.operation",
|
|
"target_field" : "operation",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"rename" : {
|
|
"field" : "parsed_json.desc",
|
|
"target_field" : "description",
|
|
"ignore_failure" : true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"kea_dhcp_pipeline" : {
|
|
"description" : "Parse logs from kea-dhcp4 to extract key fields",
|
|
"processors" : [
|
|
{
|
|
"grok" : {
|
|
"field" : "message",
|
|
"patterns" : [
|
|
"%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \\[%{DATA:service}/%{NUMBER:pid}\\.%{DATA:thread_id}\\] %{DATA:event_type} \\[hwtype=%{NUMBER:hw_type} %{MAC:mac_address}\\](?:, cid=\\[%{DATA:cid}\\])?, tid=%{DATA:transaction_id}: (?:lease %{IP:ip_address} %{GREEDYDATA:event_details})?"
|
|
],
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "service",
|
|
"value" : "kea-dhcp4",
|
|
"ignore_failure" : true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"kea_dhcp_filebeat_pipeline" : {
|
|
"description" : "Parse Kea DHCP logs from Filebeat",
|
|
"processors" : [
|
|
{
|
|
"grok" : {
|
|
"field" : "message",
|
|
"patterns" : [
|
|
"%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \\[%{DATA:service}/%{NUMBER:pid}\\.%{DATA:thread_id}\\] %{DATA:event_type} \\[hwtype=%{NUMBER:hw_type} %{MAC:mac_address}\\](?:, cid=\\[%{DATA:cid}\\])?, tid=%{DATA:transaction_id}: (?:lease %{IP:ip_address} %{GREEDYDATA:event_details})?"
|
|
],
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "service",
|
|
"value" : "kea-dhcp4",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"date" : {
|
|
"field" : "timestamp",
|
|
"formats" : [
|
|
"yyyy-MM-dd HH:mm:ss.SSS"
|
|
],
|
|
"target_field" : "@timestamp",
|
|
"ignore_failure" : true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"filter_ogboot_pipeline" : {
|
|
"description" : "Parse logs to extract http_code and desc, while preserving original message",
|
|
"processors" : [
|
|
{
|
|
"script" : {
|
|
"if" : "ctx.syslog?.identifier != 'ogboot'",
|
|
"source" : "\n ctx.debug = 'Skipped: identifier is ' + (ctx.syslog?.identifier ?: 'undefined');\n ctx.pipeline_stop = true; // Stops further processing but retains the document\n "
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "debug",
|
|
"value" : "Processed: identifier is ogboot"
|
|
}
|
|
},
|
|
{
|
|
"script" : {
|
|
"source" : "\n ctx.processed_message = ctx.message;\n "
|
|
}
|
|
},
|
|
{
|
|
"gsub" : {
|
|
"field" : "processed_message",
|
|
"pattern" : "^app\\.[A-Z]+: ",
|
|
"replacement" : "",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"gsub" : {
|
|
"field" : "processed_message",
|
|
"pattern" : "^request\\.INFO: Matched route \".*?\"\\. ",
|
|
"replacement" : "",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"json" : {
|
|
"field" : "processed_message",
|
|
"target_field" : "parsed_message",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "route",
|
|
"value" : "{{parsed_message.route}}",
|
|
"ignore_empty_value" : true,
|
|
"if" : "ctx.parsed_message?.route != null"
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "route_parameters",
|
|
"value" : "{{parsed_message.route_parameters}}",
|
|
"ignore_empty_value" : true,
|
|
"if" : "ctx.parsed_message?.route_parameters != null"
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "request_uri",
|
|
"value" : "{{parsed_message.request_uri}}",
|
|
"ignore_empty_value" : true,
|
|
"if" : "ctx.parsed_message?.request_uri != null"
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "method",
|
|
"value" : "{{parsed_message.method}}",
|
|
"ignore_empty_value" : true,
|
|
"if" : "ctx.parsed_message?.method != null"
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "http_code",
|
|
"value" : "{{parsed_message.http_code}}",
|
|
"ignore_empty_value" : true
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "description",
|
|
"value" : "{{parsed_message.desc}}",
|
|
"ignore_empty_value" : true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"ogrepo_pipeline" : {
|
|
"description" : "Pipeline to parse ogRepo logs",
|
|
"processors" : [
|
|
{
|
|
"set" : {
|
|
"field" : "debug_message",
|
|
"value" : "{{message}}"
|
|
}
|
|
},
|
|
{
|
|
"script" : {
|
|
"source" : "\n if (ctx.message != null) {\n ctx.message = ctx.message.replace(\"'\", \"\\\"\")\n }\n "
|
|
}
|
|
},
|
|
{
|
|
"json" : {
|
|
"field" : "message",
|
|
"target_field" : "parsed_json",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"remove" : {
|
|
"field" : "message",
|
|
"ignore_failure" : true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"parse_nginx_logs" : {
|
|
"description" : "Parse logs from Nginx in the 'main' log format with debug information",
|
|
"processors" : [
|
|
{
|
|
"set" : {
|
|
"field" : "debug",
|
|
"value" : "Entered parse_nginx_logs pipeline",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"gsub" : {
|
|
"field" : "log_details",
|
|
"pattern" : "^\\s+",
|
|
"replacement" : "",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"grok" : {
|
|
"field" : "log_details",
|
|
"patterns" : [
|
|
"%{IP:client_ip} %{GREEDYDATA:rest}"
|
|
],
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"grok" : {
|
|
"field" : "rest",
|
|
"patterns" : [
|
|
"- %{DATA:remote_user} \\[%{HTTPDATE:timestamp}\\] %{GREEDYDATA:rest_after_timestamp}"
|
|
],
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"grok" : {
|
|
"field" : "rest_after_timestamp",
|
|
"patterns" : [
|
|
"\"%{WORD:method} %{DATA:request_path} HTTP/%{NUMBER:http_version}\" %{NUMBER:status} %{NUMBER:body_bytes} %{GREEDYDATA:rest_referer}"
|
|
],
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"grok" : {
|
|
"field" : "rest_referer",
|
|
"patterns" : [
|
|
"\"%{DATA:referer}\" \"%{GREEDYDATA:nginx_user_agent}\""
|
|
],
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"date" : {
|
|
"field" : "timestamp",
|
|
"formats" : [
|
|
"dd/MMM/yyyy:HH:mm:ss Z"
|
|
],
|
|
"target_field" : "@timestamp",
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"remove" : {
|
|
"field" : [
|
|
"rest"
|
|
],
|
|
"ignore_missing" : true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"kea_dhcp_parse_pipeline" : {
|
|
"description" : "Parse Kea DHCP logs for detailed information",
|
|
"processors" : [
|
|
{
|
|
"grok" : {
|
|
"field" : "message",
|
|
"patterns" : [
|
|
"%{TIMESTAMP_ISO8601:timestamp} +%{LOGLEVEL:log_level} \\[%{DATA:source}/%{NUMBER:pid}.%{NUMBER:thread_id}\\] %{WORD:message_id} \\[%{DATA:hwtype}\\], cid=%{DATA:cid}, tid=%{DATA:tid}: lease %{IP:lease} has been allocated for %{NUMBER:lease_duration} seconds"
|
|
],
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "service",
|
|
"value" : "kea-dhcp4",
|
|
"ignore_failure" : true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"json_parse_with_replacement" : {
|
|
"description" : "Replace single quotes with double quotes and parse JSON",
|
|
"processors" : [
|
|
{
|
|
"script" : {
|
|
"source" : "\n ctx.message = ctx.message.replace(\"'\", \"\\\"\");\n "
|
|
}
|
|
},
|
|
{
|
|
"json" : {
|
|
"field" : "message",
|
|
"target_field" : "parsed_json"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"tftp_parse_pipeline" : {
|
|
"description" : "Parse logs from in.tftpd to extract filename and client IP",
|
|
"processors" : [
|
|
{
|
|
"grok" : {
|
|
"field" : "message",
|
|
"patterns" : [
|
|
"RRQ from %{HOSTNAME:client_ip} filename %{GREEDYDATA:filename}"
|
|
],
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "service",
|
|
"value" : "tftpd",
|
|
"ignore_failure" : true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"filter_tftp_pipeline" : {
|
|
"description" : "Parse logs from in.tftpd to extract filename and client IP",
|
|
"processors" : [
|
|
{
|
|
"grok" : {
|
|
"field" : "message",
|
|
"patterns" : [
|
|
"RRQ from %{HOSTNAME:client_ip} filename %{GREEDYDATA:filename}"
|
|
],
|
|
"ignore_failure" : true
|
|
}
|
|
},
|
|
{
|
|
"set" : {
|
|
"field" : "service_name",
|
|
"value" : "tftpd",
|
|
"ignore_failure" : true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"copy-message-pipeline" : {
|
|
"description" : "Pipeline que copia el campo message a message_raw",
|
|
"processors" : [
|
|
{
|
|
"set" : {
|
|
"field" : "message_raw",
|
|
"value" : "{{message}}"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
|