From 5f228f16017613197f8f8fb946c4dd78d42783cc Mon Sep 17 00:00:00 2001 From: lgromero Date: Thu, 31 Jul 2025 14:01:09 +0200 Subject: [PATCH] refs #2550 updates dashboards omitting some columns and adds ogcre parse json to master pipeline --- .../resources/dashboards/ogboot-logs.json | 5 +- .../resources/dashboards/ogcore-logs.json | 190 ++++++++++++++---- .../resources/dashboards/ogdhcp-logs.json | 4 +- .../resources/dashboards/ogrepo-logs.json | 28 ++- etc/opensearch/pipelines.json | 3 +- 5 files changed, 182 insertions(+), 48 deletions(-) diff --git a/etc/grafana/resources/dashboards/ogboot-logs.json b/etc/grafana/resources/dashboards/ogboot-logs.json index 850cefe..63d4a08 100644 --- a/etc/grafana/resources/dashboards/ogboot-logs.json +++ b/etc/grafana/resources/dashboards/ogboot-logs.json @@ -343,7 +343,7 @@ ], "datasource": { "type": "grafana-opensearch-datasource", - "uid": "OpenSearch-Opengnsys" + "uid": "${DS_OPENSEARCH}" }, "format": "table", "luceneQueryType": "Logs", @@ -536,7 +536,6 @@ "type": "table" } ], - "refresh": "5s", "schemaVersion": 41, "tags": [], "templating": { @@ -552,4 +551,4 @@ "uid": "ogboot-logs", "version": 7, "weekStart": "" -} +} \ No newline at end of file diff --git a/etc/grafana/resources/dashboards/ogcore-logs.json b/etc/grafana/resources/dashboards/ogcore-logs.json index 733d320..16f5b49 100644 --- a/etc/grafana/resources/dashboards/ogcore-logs.json +++ b/etc/grafana/resources/dashboards/ogcore-logs.json @@ -227,6 +227,42 @@ "value": 217 } ] + }, + { + "matcher": { + "id": "byName", + "options": "parsed_message.desc" + }, + "properties": [ + { + "id": "custom.width", + "value": 1266 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "fecha" + }, + "properties": [ + { + "id": "custom.width", + "value": 427 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nivel de log" + }, + "properties": [ + { + "id": "custom.width", + "value": 115 + } + ] } ] }, @@ -266,7 +302,7 @@ ], "datasource": { "type": "grafana-opensearch-datasource", - "uid": "OpenSearch-Opengnsys" + "uid": "${DS_OPENSEARCH}" }, "format": "table", "luceneQueryType": "Logs", @@ -276,7 +312,7 @@ "type": "logs" } ], - "query": "syslog.identifier: \"ogcore\"", + "query": "syslog.identifier:\"ogcore\" AND (parsed_message.severity:\"INFO\" OR parsed_message.severity:\"WARNING\" OR parsed_message.severity:\"ERROR\")", "queryType": "lucene", "refId": "A", "timeField": "@timestamp" @@ -313,7 +349,10 @@ "data_decoded.timestamp": true, "data_decoded.tpc": true, "data_json": true, + "debug": true, "ecs.version": true, + "event.created": true, + "event.kind": true, "host.architecture": true, "host.containerized": true, "host.hostname": true, @@ -329,15 +368,63 @@ "host.os.type": true, "host.os.version": true, "input.type": true, + "journald.audit.login_uid": true, + "journald.audit.session": true, + "journald.custom.runtime_scope": true, + "journald.custom.selinux_context": true, + "journald.custom.syslog_raw": true, + "journald.custom.syslog_timestamp": true, + "journald.gid": true, + "journald.host.boot_id": true, + "journald.pid": true, + "journald.process.capabilites": true, + "journald.process.command_line": true, + "journald.process.executable": true, + "journald.process.name": true, + "journald.uid": true, "log.file.path": true, "log.offset": true, + "log.syslog.facility.name": true, + "log.syslog.priority": true, "message": true, "message_decoded.function": true, "message_decoded.in_oglive": true, "message_decoded.message": false, "message_decoded.threadName": true, "message_decoded.timestamp": true, - "message_raw": true + "message_raw": true, + "parsed_message": true, + "parsed_message.component": true, + "parsed_message.datetime": true, + "parsed_message.desc": false, + "parsed_message.operation": true, + "parsed_message.params": true, + "parsed_message.params.cache-adapter": true, + "parsed_message.params.exception": true, + "parsed_message.params.iph": true, + "parsed_message.params.key": true, + "parsed_message.params.method": true, + "parsed_message.params.request_uri": true, + "parsed_message.params.route": true, + "parsed_message.params.route_parameters._controller": true, + "parsed_message.params.route_parameters._route": true, + "parsed_message.params.timestamp": true, + "parsed_message.severity": false, + "process.args": true, + "process.args_count": true, + "process.command_line": true, + "process.pid": true, + "syslog.facility": true, + "syslog.identifier": true, + "syslog.pid": true, + "syslog.priority": true, + "systemd.cgroup": true, + "systemd.invocation_id": true, + "systemd.slice": true, + "systemd.transport": true, + "systemd.unit": true, + "user.group.id": true, + "user.id": true }, "includeByName": {}, "indexByName": { @@ -352,40 +439,69 @@ "agent.name": 8, "agent.type": 9, "agent.version": 10, - "data_decoded.iph": 11, - "data_decoded.timestamp": 13, - "data_json": 35, - "ecs.version": 14, - "host.architecture": 15, - "host.containerized": 16, - "host.hostname": 17, - "host.id": 19, - "host.ip": 18, - "host.mac": 20, - "host.name": 21, - "host.os.codename": 22, - "host.os.family": 23, - "host.os.kernel": 24, - "host.os.name": 25, - "host.os.platform": 26, - "host.os.type": 27, - "host.os.version": 28, - "input.type": 29, - "log.file.path": 30, - "log.offset": 31, - "message": 38, - "message_decoded.function": 32, - "message_decoded.message": 33, - "message_decoded.severity": 12, - "message_decoded.threadName": 34, - "message_decoded.timestamp": 36, - "message_raw": 37 + "debug": 16, + "ecs.version": 11, + "event.created": 17, + "event.kind": 18, + "host.hostname": 12, + "host.id": 13, + "host.name": 14, + "journald.audit.login_uid": 19, + "journald.audit.session": 20, + "journald.custom.runtime_scope": 21, + "journald.custom.selinux_context": 22, + "journald.custom.syslog_timestamp": 23, + "journald.gid": 24, + "journald.host.boot_id": 25, + "journald.pid": 26, + "journald.process.capabilites": 27, + "journald.process.command_line": 28, + "journald.process.executable": 29, + "journald.process.name": 30, + "journald.uid": 31, + "log.syslog.facility.name": 32, + "log.syslog.priority": 33, + "message": 15, + "parsed_message.component": 34, + "parsed_message.datetime": 35, + "parsed_message.desc": 38, + "parsed_message.operation": 37, + "parsed_message.params": 39, + "parsed_message.params.cache-adapter": 40, + "parsed_message.params.exception": 41, + "parsed_message.params.iph": 42, + "parsed_message.params.key": 43, + "parsed_message.params.method": 44, + "parsed_message.params.request_uri": 45, + "parsed_message.params.route": 46, + "parsed_message.params.route_parameters._controller": 47, + "parsed_message.params.route_parameters._route": 48, + "parsed_message.params.timestamp": 49, + "parsed_message.severity": 36, + "process.args": 50, + "process.args_count": 51, + "process.command_line": 52, + "process.pid": 53, + "syslog.facility": 54, + "syslog.identifier": 55, + "syslog.pid": 56, + "syslog.priority": 57, + "systemd.cgroup": 58, + "systemd.invocation_id": 59, + "systemd.slice": 60, + "systemd.transport": 61, + "systemd.unit": 62, + "user.group.id": 63, + "user.id": 64 }, "renameByName": { - "@timestamp": "fecha", + "@timestamp": "Fecha", "agent.type": "", "message_decoded.message": "message", - "message_decoded.severity": "log level" + "message_decoded.severity": "log level", + "parsed_message.desc": "Descripción", + "parsed_message.operation": "", + "parsed_message.severity": "Nivel de log" } } } @@ -399,13 +515,13 @@ "list": [] }, "time": { - "from": "2025-07-30T10:27:32.439Z", - "to": "2025-07-30T10:37:32.439Z" + "from": "now-5m", + "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "ogcore-logs", "uid": "ogcore-logs", - "version": 3, + "version": 4, "weekStart": "" -} +} \ No newline at end of file diff --git a/etc/grafana/resources/dashboards/ogdhcp-logs.json b/etc/grafana/resources/dashboards/ogdhcp-logs.json index a2e9288..5098cb0 100644 --- a/etc/grafana/resources/dashboards/ogdhcp-logs.json +++ b/etc/grafana/resources/dashboards/ogdhcp-logs.json @@ -350,7 +350,7 @@ ], "datasource": { "type": "grafana-opensearch-datasource", - "uid": "OpenSearch-Opengnsys" + "uid": "${DS_OPENSEARCH}" }, "format": "table", "luceneQueryType": "Logs", @@ -559,4 +559,4 @@ "uid": "ogdhcp-logs", "version": 3, "weekStart": "" -} +} \ No newline at end of file diff --git a/etc/grafana/resources/dashboards/ogrepo-logs.json b/etc/grafana/resources/dashboards/ogrepo-logs.json index 8908b15..998b149 100644 --- a/etc/grafana/resources/dashboards/ogrepo-logs.json +++ b/etc/grafana/resources/dashboards/ogrepo-logs.json @@ -362,7 +362,7 @@ ], "datasource": { "type": "grafana-opensearch-datasource", - "uid": "OpenSearch-Opengnsys" + "uid": "${DS_OPENSEARCH}" }, "format": "table", "luceneQueryType": "Logs", @@ -372,7 +372,7 @@ "type": "logs" } ], - "query": "syslog.identifier: \"ogrepo-api\"", + "query": "syslog.identifier: \"ogrepo-api\" AND host.hostname: $hostname", "queryType": "lucene", "refId": "A", "timeField": "@timestamp" @@ -560,7 +560,25 @@ "schemaVersion": 41, "tags": [], "templating": { - "list": [] + "list": [ + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"host.hostname\", \"query\": \"syslog.identifier:\\\"ogrepo-api\\\"\"}\n", + "includeAll": true, + "label": "Host", + "multi": true, + "name": "hostname", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"host.hostname\", \"query\": \"syslog.identifier:\\\"ogrepo-api\\\"\"}\n", + "refresh": 1, + "regex": "", + "type": "query" + } + ] }, "time": { "from": "now-30m", @@ -570,6 +588,6 @@ "timezone": "browser", "title": "ogrepo-logs", "uid": "ogrepo-logs", - "version": 2, + "version": 4, "weekStart": "" -} +} \ No newline at end of file diff --git a/etc/opensearch/pipelines.json b/etc/opensearch/pipelines.json index 380674b..cd3df45 100644 --- a/etc/opensearch/pipelines.json +++ b/etc/opensearch/pipelines.json @@ -90,6 +90,7 @@ } ] }, + "master_pipeline" : { "description" : "Master pipeline to route logs based on syslog.identifier", "processors" : [ @@ -141,7 +142,7 @@ "set" : { "field" : "debug", "value" : "No matching pipeline, skipping further processing.", - "if" : "ctx.syslog?.identifier != 'in.tftpd' && ctx.syslog?.identifier != 'ogboot' && ctx.syslog?.identifier != 'kea-dhcp4' && ctx.syslog?.identifier != 'ogrepo-api' && ctx.syslog?.identifier != 'docker'" + "if" : "ctx.syslog?.identifier != 'in.tftpd' && ctx.syslog?.identifier != 'ogboot' && ctx.syslog?.identifier != 'kea-dhcp4' && ctx.syslog?.identifier != 'ogrepo-api' && ctx.syslog?.identifier != 'ogcore' && ctx.syslog?.identifier != 'docker'" } } ]