From b957867d419256e88ed1666ec17e4d0844569ed7 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 16 Apr 2025 07:56:30 +0000 Subject: [PATCH] refs #1806 Removes id when import grafana dashboard. Use absolute route in opensearch yml. Add opensearch pipeline to import. Adds saved searches and import in opensearch --- .../opensearch_dashboards.yml | 8 +- .../saved_searches.ndjson | 10 + etc/opensearch/opensearch.yml | 26 +- etc/opensearch/pipelines.json | 597 ++++++++++++++++++ script/import_grafana.sh | 7 +- script/oglog_installer.sh | 135 +++- 6 files changed, 757 insertions(+), 26 deletions(-) create mode 100644 etc/opensearch-dashboards/saved_searches.ndjson create mode 100644 etc/opensearch/pipelines.json diff --git a/etc/opensearch-dashboards/opensearch_dashboards.yml b/etc/opensearch-dashboards/opensearch_dashboards.yml index 5a833e0..ca51024 100644 --- a/etc/opensearch-dashboards/opensearch_dashboards.yml +++ b/etc/opensearch-dashboards/opensearch_dashboards.yml @@ -3,10 +3,10 @@ opensearch.hosts: ["https://oglog-os.mytld:9200"] opensearch.username: "admin" opensearch.password: "{{OPENSEARCH_INITIAL_ADMIN_PASSWORD}}" server.ssl.enabled: true -server.ssl.certificate: oglog-osdb.mytld.crt.pem -server.ssl.key: oglog-osdb.mytld.key.pem -opensearch.ssl.certificate: oglog-osdb.mytld.crt.pem -opensearch.ssl.key: oglog-osdb.mytld.key.pem +server.ssl.certificate: /etc/opensearch-dashboards/oglog-osdb.mytld.crt.pem +server.ssl.key: /etc/opensearch-dashboards/oglog-osdb.mytld.key.pem +opensearch.ssl.certificate: /etc/opensearch-dashboards/oglog-osdb.mytld.crt.pem +opensearch.ssl.key: /etc/opensearch-dashboards/oglog-osdb.mytld.key.pem opensearch.ssl.verificationMode: full opensearch.ssl.certificateAuthorities: ["/etc/ssl/certs/ca.crt.pem"] opensearch.ssl.alwaysPresentCertificate: true diff --git a/etc/opensearch-dashboards/saved_searches.ndjson b/etc/opensearch-dashboards/saved_searches.ndjson new file mode 100644 index 0000000..b3196bd --- /dev/null +++ b/etc/opensearch-dashboards/saved_searches.ndjson @@ -0,0 +1,10 @@ +{"attributes":{"columns":["agent.name","host.ip","host.mac","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"log.file.path\",\"params\":{\"query\":\"/var/log/opengnsys.log\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"log.file.path\":\"/var/log/opengnsys.log\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Ogagent","version":1},"id":"06a268e0-d3d0-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-16T06:08:03.537Z","version":"WzQ5LDJd"} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"syslog.identifier\",\"params\":{\"query\":\"docker\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"docker\"}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"container.name\",\"params\":{\"query\":\"ogcore-nginx\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index\"},\"query\":{\"match_phrase\":{\"container.name\":\"ogcore-nginx\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"ogcore-nginx","version":1},"id":"0c6388d0-d3d1-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-16T06:13:48.124Z","version":"WzUzLDJd"} +{"attributes":{"columns":["agent.name","host.ip","host.mac","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30m\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"log.file.path\",\"params\":{\"query\":\"/var/log/opengnsys.log\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"log.file.path\":\"/var/log/opengnsys.log\"}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"agent.name\",\"params\":{\"query\":\"pc-modelo-ubuntu24-2\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index\"},\"query\":{\"match_phrase\":{\"agent.name\":\"pc-modelo-ubuntu24-2\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Ogagent - agent name","version":1},"id":"353bcbf0-d7d2-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-21T08:32:10.797Z","version":"WzU1LDJd"} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Kea Dhcp","version":1},"id":"39976990-d3cf-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-16T06:00:44.967Z","version":"WzQzLDJd"} +{"attributes":{"columns":["parsed_message.severity","parsed_message.operation","parsed_message.http_code","parsed_message.desc","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"syslog.identifier\",\"params\":{\"query\":\"ogdhcp\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"ogdhcp\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Ogdhcp","version":1},"id":"8542fc60-d249-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-14T07:33:35.272Z","version":"WzQyLDJd"} +{"attributes":{"columns":["parsed_message.severity","parsed_message.method","parsed_message.request_uri","parsed_message.operation","parsed_message.http_code","parsed_message.desc","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"syslog.identifier\",\"negate\":false,\"params\":{\"query\":\"ogboot\"},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"ogboot\"}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Ogboot","version":1},"id":"8b2a50b0-d244-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-14T07:00:24.803Z","version":"WzM1LDJd"} +{"attributes":{"columns":["parsed_json.severity","parsed_json.component","parsed_json.operation","parsed_json.desc"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"syslog.identifier\",\"params\":{\"query\":\"docker\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"docker\"}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"container.name\",\"params\":{\"query\":\"ogcore-php\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index\"},\"query\":{\"match_phrase\":{\"container.name\":\"ogcore-php\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"ogcore-php","version":1},"id":"abe87370-d188-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-13T08:30:40.160Z","version":"WzksMl0="} +{"exportedCount":7,"missingRefCount":0,"missingReferences":[]} + + diff --git a/etc/opensearch/opensearch.yml b/etc/opensearch/opensearch.yml index 89eb979..79733db 100644 --- a/etc/opensearch/opensearch.yml +++ b/etc/opensearch/opensearch.yml @@ -1,8 +1,32 @@ +# WARNING: revise all the lines below before you go into production network.host: "{{IP_MAQUINA}}" +plugins.security.ssl.transport.pemcert_filepath: esnode.pem +plugins.security.ssl.transport.pemkey_filepath: esnode-key.pem +plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem +plugins.security.ssl.transport.enforce_hostname_verification: false +plugins.security.ssl.http.enabled: true plugins.security.ssl.http.pemcert_filepath: oglog-os.mytld.crt.pem plugins.security.ssl.http.pemkey_filepath: oglog-os.mytld.key.pem plugins.security.ssl.http.pemtrustedcas_filepath: ca.crt.pem - +plugins.security.allow_unsafe_democertificates: true +plugins.security.allow_default_init_securityindex: true +plugins.security.authcz.admin_dn: ['CN=kirk,OU=client,O=client,L=test,C=de'] +plugins.security.audit.type: internal_opensearch +plugins.security.enable_snapshot_restore_privilege: true +plugins.security.check_snapshot_restore_write_privileges: true +plugins.security.restapi.roles_enabled: [all_access, security_rest_api_access] +plugins.security.system_indices.enabled: true +plugins.security.system_indices.indices: [.plugins-ml-agent, .plugins-ml-config, .plugins-ml-connector, + .plugins-ml-controller, .plugins-ml-model-group, .plugins-ml-model, .plugins-ml-task, + .plugins-ml-conversation-meta, .plugins-ml-conversation-interactions, .plugins-ml-memory-meta, + .plugins-ml-memory-message, .plugins-ml-stop-words, .opendistro-alerting-config, + .opendistro-alerting-alert*, .opendistro-anomaly-results*, .opendistro-anomaly-detector*, + .opendistro-anomaly-checkpoints, .opendistro-anomaly-detection-state, .opendistro-reports-*, + .opensearch-notifications-*, .opensearch-notebooks, .opensearch-observability, .ql-datasources, + .opendistro-asynchronous-search-response*, .replication-metadata-store, .opensearch-knn-models, + .geospatial-ip2geo-data*, .plugins-flow-framework-config, .plugins-flow-framework-templates, + .plugins-flow-framework-state] +node.max_local_storage_nodes: 3 discovery.type: single-node compatibility.override_main_response_version: true plugins.security.ssl.http.clientauth_mode: REQUIRE diff --git a/etc/opensearch/pipelines.json b/etc/opensearch/pipelines.json new file mode 100644 index 0000000..6f89c91 --- /dev/null +++ b/etc/opensearch/pipelines.json @@ -0,0 +1,597 @@ +{ + "filter_ogdhcp_pipeline" : { + "description" : "Parse logs to extract http_code and desc, while preserving original message", + "processors" : [ + { + "script" : { + "if" : "ctx.syslog?.identifier != 'ogdhcp'", + "source" : "\n ctx.debug = 'Skipped: identifier is ' + (ctx.syslog?.identifier ?: 'undefined');\n ctx.pipeline_stop = true; // Stops further processing but retains the document\n " + } + }, + { + "set" : { + "field" : "debug", + "value" : "Processed: identifier is ogdhcp" + } + }, + { + "script" : { + "source" : "\n ctx.processed_message = ctx.message;\n " + } + }, + { + "gsub" : { + "field" : "processed_message", + "pattern" : "^app\\.[A-Z]+: ", + "replacement" : "", + "ignore_failure" : true + } + }, + { + "gsub" : { + "field" : "processed_message", + "pattern" : "^request\\.INFO: Matched route \".*?\"\\. ", + "replacement" : "", + "ignore_failure" : true + } + }, + { + "json" : { + "field" : "processed_message", + "target_field" : "parsed_message", + "ignore_failure" : true + } + }, + { + "set" : { + "field" : "route", + "value" : "{{parsed_message.route}}", + "ignore_empty_value" : true, + "if" : "ctx.parsed_message?.route != null" + } + }, + { + "set" : { + "field" : "route_parameters", + "value" : "{{parsed_message.route_parameters}}", + "ignore_empty_value" : true, + "if" : "ctx.parsed_message?.route_parameters != null" + } + }, + { + "set" : { + "field" : "request_uri", + "value" : "{{parsed_message.request_uri}}", + "ignore_empty_value" : true, + "if" : "ctx.parsed_message?.request_uri != null" + } + }, + { + "set" : { + "field" : "method", + "value" : "{{parsed_message.method}}", + "ignore_empty_value" : true, + "if" : "ctx.parsed_message?.method != null" + } + }, + { + "set" : { + "field" : "http_code", + "value" : "{{parsed_message.http_code}}", + "ignore_empty_value" : true + } + }, + { + "set" : { + "field" : "description", + "value" : "{{parsed_message.desc}}", + "ignore_empty_value" : true + } + } + ] + }, + "master_pipeline" : { + "description" : "Master pipeline to route logs based on syslog.identifier", + "processors" : [ + { + "pipeline" : { + "name" : "filter_tftp_pipeline", + "if" : "ctx.syslog?.identifier == 'in.tftpd'" + } + }, + { + "pipeline" : { + "name" : "filter_ogboot_pipeline", + "if" : "ctx.syslog?.identifier == 'ogboot'" + } + }, + { + "pipeline" : { + "name" : "filter_ogdhcp_pipeline", + "if" : "ctx.syslog?.identifier == 'ogdhcp'" + } + }, + { + "pipeline" : { + "name" : "kea_dhcp_pipeline", + "if" : "ctx.syslog?.identifier == 'kea-dhcp4'" + } + }, + { + "pipeline" : { + "name" : "ogrepo_pipeline", + "if" : "ctx.syslog?.identifier == 'ogrepo-api'" + } + }, + { + "pipeline" : { + "name" : "docker_logs_pipeline", + "if" : "ctx.syslog?.identifier == 'docker'" + } + }, + { + "set" : { + "field" : "debug", + "value" : "No matching pipeline, skipping further processing.", + "if" : "ctx.syslog?.identifier != 'in.tftpd' && ctx.syslog?.identifier != 'ogboot' && ctx.syslog?.identifier != 'kea-dhcp4' && ctx.syslog?.identifier != 'ogrepo-api' && ctx.syslog?.identifier != 'docker'" + } + } + ] + }, + "json_parse_pipeline" : { + "description" : "Parse JSON payload from logs", + "processors" : [ + { + "json" : { + "field" : "message", + "target_field" : "parsed_json", + "ignore_failure" : true + } + } + ] + }, + "docker_logs_pipeline" : { + "description" : "Parse Docker logs and route based on container name", + "processors" : [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{DATA:container.name}\\s*\\|%{GREEDYDATA:log_details}" + ], + "ignore_failure" : true + } + }, + { + "pipeline" : { + "name" : "parse_nginx_logs", + "if" : "ctx.container?.name == 'ogcore-nginx'", + "ignore_failure" : true + } + }, + { + "json" : { + "field" : "log_details", + "target_field" : "parsed_json", + "ignore_failure" : true + } + } + ] + }, + "json_parse_with_replacement_debug" : { + "description" : "Debug replacement of single quotes with double quotes and parse JSON", + "processors" : [ + { + "script" : { + "source" : "\n ctx.message = ctx.message.replace(\"'\", \"\\\"\");\n " + } + }, + { + "set" : { + "field" : "debug_message", + "value" : "{{ message }}" + } + }, + { + "json" : { + "field" : "message", + "target_field" : "parsed_json", + "ignore_failure" : true + } + } + ] + }, + "ogrepo_parse_pipeline" : { + "description" : "Parse ogRepo logs for detailed JSON information", + "processors" : [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{TIMESTAMP_ISO8601:timestamp} %{DATA:hostname} %{DATA:service}\\[%{NUMBER:pid}\\]: %{GREEDYDATA:json_payload}" + ], + "ignore_failure" : true + } + }, + { + "json" : { + "field" : "json_payload", + "target_field" : "parsed_json", + "ignore_failure" : true + } + }, + { + "rename" : { + "field" : "parsed_json.component", + "target_field" : "component", + "ignore_failure" : true + } + }, + { + "rename" : { + "field" : "parsed_json.severity", + "target_field" : "severity", + "ignore_failure" : true + } + }, + { + "rename" : { + "field" : "parsed_json.http_code", + "target_field" : "http_code", + "ignore_failure" : true + } + }, + { + "rename" : { + "field" : "parsed_json.operation", + "target_field" : "operation", + "ignore_failure" : true + } + }, + { + "rename" : { + "field" : "parsed_json.desc", + "target_field" : "description", + "ignore_failure" : true + } + } + ] + }, + "kea_dhcp_pipeline" : { + "description" : "Parse logs from kea-dhcp4 to extract key fields", + "processors" : [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \\[%{DATA:service}/%{NUMBER:pid}\\.%{DATA:thread_id}\\] %{DATA:event_type} \\[hwtype=%{NUMBER:hw_type} %{MAC:mac_address}\\](?:, cid=\\[%{DATA:cid}\\])?, tid=%{DATA:transaction_id}: (?:lease %{IP:ip_address} %{GREEDYDATA:event_details})?" + ], + "ignore_failure" : true + } + }, + { + "set" : { + "field" : "service", + "value" : "kea-dhcp4", + "ignore_failure" : true + } + } + ] + }, + "kea_dhcp_filebeat_pipeline" : { + "description" : "Parse Kea DHCP logs from Filebeat", + "processors" : [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \\[%{DATA:service}/%{NUMBER:pid}\\.%{DATA:thread_id}\\] %{DATA:event_type} \\[hwtype=%{NUMBER:hw_type} %{MAC:mac_address}\\](?:, cid=\\[%{DATA:cid}\\])?, tid=%{DATA:transaction_id}: (?:lease %{IP:ip_address} %{GREEDYDATA:event_details})?" + ], + "ignore_failure" : true + } + }, + { + "set" : { + "field" : "service", + "value" : "kea-dhcp4", + "ignore_failure" : true + } + }, + { + "date" : { + "field" : "timestamp", + "formats" : [ + "yyyy-MM-dd HH:mm:ss.SSS" + ], + "target_field" : "@timestamp", + "ignore_failure" : true + } + } + ] + }, + "filter_ogboot_pipeline" : { + "description" : "Parse logs to extract http_code and desc, while preserving original message", + "processors" : [ + { + "script" : { + "if" : "ctx.syslog?.identifier != 'ogboot'", + "source" : "\n ctx.debug = 'Skipped: identifier is ' + (ctx.syslog?.identifier ?: 'undefined');\n ctx.pipeline_stop = true; // Stops further processing but retains the document\n " + } + }, + { + "set" : { + "field" : "debug", + "value" : "Processed: identifier is ogboot" + } + }, + { + "script" : { + "source" : "\n ctx.processed_message = ctx.message;\n " + } + }, + { + "gsub" : { + "field" : "processed_message", + "pattern" : "^app\\.[A-Z]+: ", + "replacement" : "", + "ignore_failure" : true + } + }, + { + "gsub" : { + "field" : "processed_message", + "pattern" : "^request\\.INFO: Matched route \".*?\"\\. ", + "replacement" : "", + "ignore_failure" : true + } + }, + { + "json" : { + "field" : "processed_message", + "target_field" : "parsed_message", + "ignore_failure" : true + } + }, + { + "set" : { + "field" : "route", + "value" : "{{parsed_message.route}}", + "ignore_empty_value" : true, + "if" : "ctx.parsed_message?.route != null" + } + }, + { + "set" : { + "field" : "route_parameters", + "value" : "{{parsed_message.route_parameters}}", + "ignore_empty_value" : true, + "if" : "ctx.parsed_message?.route_parameters != null" + } + }, + { + "set" : { + "field" : "request_uri", + "value" : "{{parsed_message.request_uri}}", + "ignore_empty_value" : true, + "if" : "ctx.parsed_message?.request_uri != null" + } + }, + { + "set" : { + "field" : "method", + "value" : "{{parsed_message.method}}", + "ignore_empty_value" : true, + "if" : "ctx.parsed_message?.method != null" + } + }, + { + "set" : { + "field" : "http_code", + "value" : "{{parsed_message.http_code}}", + "ignore_empty_value" : true + } + }, + { + "set" : { + "field" : "description", + "value" : "{{parsed_message.desc}}", + "ignore_empty_value" : true + } + } + ] + }, + "ogrepo_pipeline" : { + "description" : "Pipeline to parse ogRepo logs", + "processors" : [ + { + "set" : { + "field" : "debug_message", + "value" : "{{message}}" + } + }, + { + "script" : { + "source" : "\n if (ctx.message != null) {\n ctx.message = ctx.message.replace(\"'\", \"\\\"\")\n }\n " + } + }, + { + "json" : { + "field" : "message", + "target_field" : "parsed_json", + "ignore_failure" : true + } + }, + { + "remove" : { + "field" : "message", + "ignore_failure" : true + } + } + ] + }, + "parse_nginx_logs" : { + "description" : "Parse logs from Nginx in the 'main' log format with debug information", + "processors" : [ + { + "set" : { + "field" : "debug", + "value" : "Entered parse_nginx_logs pipeline", + "ignore_failure" : true + } + }, + { + "gsub" : { + "field" : "log_details", + "pattern" : "^\\s+", + "replacement" : "", + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "log_details", + "patterns" : [ + "%{IP:client_ip} %{GREEDYDATA:rest}" + ], + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "rest", + "patterns" : [ + "- %{DATA:remote_user} \\[%{HTTPDATE:timestamp}\\] %{GREEDYDATA:rest_after_timestamp}" + ], + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "rest_after_timestamp", + "patterns" : [ + "\"%{WORD:method} %{DATA:request_path} HTTP/%{NUMBER:http_version}\" %{NUMBER:status} %{NUMBER:body_bytes} %{GREEDYDATA:rest_referer}" + ], + "ignore_failure" : true + } + }, + { + "grok" : { + "field" : "rest_referer", + "patterns" : [ + "\"%{DATA:referer}\" \"%{GREEDYDATA:nginx_user_agent}\"" + ], + "ignore_failure" : true + } + }, + { + "date" : { + "field" : "timestamp", + "formats" : [ + "dd/MMM/yyyy:HH:mm:ss Z" + ], + "target_field" : "@timestamp", + "ignore_failure" : true + } + }, + { + "remove" : { + "field" : [ + "rest" + ], + "ignore_missing" : true + } + } + ] + }, + "kea_dhcp_parse_pipeline" : { + "description" : "Parse Kea DHCP logs for detailed information", + "processors" : [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "%{TIMESTAMP_ISO8601:timestamp} +%{LOGLEVEL:log_level} \\[%{DATA:source}/%{NUMBER:pid}.%{NUMBER:thread_id}\\] %{WORD:message_id} \\[%{DATA:hwtype}\\], cid=%{DATA:cid}, tid=%{DATA:tid}: lease %{IP:lease} has been allocated for %{NUMBER:lease_duration} seconds" + ], + "ignore_failure" : true + } + }, + { + "set" : { + "field" : "service", + "value" : "kea-dhcp4", + "ignore_failure" : true + } + } + ] + }, + "json_parse_with_replacement" : { + "description" : "Replace single quotes with double quotes and parse JSON", + "processors" : [ + { + "script" : { + "source" : "\n ctx.message = ctx.message.replace(\"'\", \"\\\"\");\n " + } + }, + { + "json" : { + "field" : "message", + "target_field" : "parsed_json" + } + } + ] + }, + "tftp_parse_pipeline" : { + "description" : "Parse logs from in.tftpd to extract filename and client IP", + "processors" : [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "RRQ from %{HOSTNAME:client_ip} filename %{GREEDYDATA:filename}" + ], + "ignore_failure" : true + } + }, + { + "set" : { + "field" : "service", + "value" : "tftpd", + "ignore_failure" : true + } + } + ] + }, + "filter_tftp_pipeline" : { + "description" : "Parse logs from in.tftpd to extract filename and client IP", + "processors" : [ + { + "grok" : { + "field" : "message", + "patterns" : [ + "RRQ from %{HOSTNAME:client_ip} filename %{GREEDYDATA:filename}" + ], + "ignore_failure" : true + } + }, + { + "set" : { + "field" : "service_name", + "value" : "tftpd", + "ignore_failure" : true + } + } + ] + }, + "copy-message-pipeline" : { + "description" : "Pipeline que copia el campo message a message_raw", + "processors" : [ + { + "set" : { + "field" : "message_raw", + "value" : "{{message}}" + } + } + ] + } +} + diff --git a/script/import_grafana.sh b/script/import_grafana.sh index 60ff771..870e99f 100755 --- a/script/import_grafana.sh +++ b/script/import_grafana.sh @@ -29,13 +29,14 @@ fi echo "Importando dashboards..." for f in "$RESOURCE_DIR/dashboards"/*.json; do echo "Importando $(basename "$f")" + + jq 'del(.dashboard.id) | {dashboard: .dashboard, overwrite: true}' "$f" | \ curl -s -X POST "$GRAFANA_URL/api/dashboards/db" \ -H "Authorization: Bearer $API_TOKEN" \ -H "Content-Type: application/json" \ - --data-binary "@$f" > /dev/null + --data-binary @- done echo "Dashboards importados." - # Importar alertas if [ -f "$RESOURCE_DIR/alerts/alert-rules.json" ]; then echo "Importando alert rules..." @@ -43,7 +44,7 @@ if [ -f "$RESOURCE_DIR/alerts/alert-rules.json" ]; then curl -s -X POST "$GRAFANA_URL/api/v1/provisioning/alert-rules" \ -H "Authorization: Bearer $API_TOKEN" \ -H "Content-Type: application/json" \ - -d "$alert" > /dev/null + -d "$alert" done echo "Alertas importadas." fi diff --git a/script/oglog_installer.sh b/script/oglog_installer.sh index ef2f141..89e4111 100755 --- a/script/oglog_installer.sh +++ b/script/oglog_installer.sh @@ -22,6 +22,9 @@ uptime | tee -a "$LOGFILE" # Inicio del cronómetro SECONDS=0 +# Instalación de paquetes necesarios +apt-get update +apt-get install -y nfs-common # Montar servidor NFS NFS_SERVER="ognartefactos.evlt.uma.es" NFS_PATH="/" @@ -64,19 +67,44 @@ if [[ ${#OPENSEARCH_INITIAL_ADMIN_PASSWORD} -lt 12 || \ exit 1 fi -# Actualizar /etc/hosts -cat >> /etc/hosts < /etc/apt/sources.list.d/opensearch.list +echo "deb [signed-by=/usr/share/keyrings/opensearch-keyring.gpg] https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.x/apt stable main" > /etc/apt/sources.list.d/opensearch-dashboards.list + +# Añadir repositorio y clave GPG para Grafana +curl -fsSL https://apt.grafana.com/gpg.key | gpg --dearmor -o /usr/share/keyrings/grafana.gpg +echo "deb [signed-by=/usr/share/keyrings/grafana.gpg] https://apt.grafana.com stable main" > /etc/apt/sources.list.d/grafana.list + + # Instalación de paquetes necesarios apt-get update -apt-get install -y apt-transport-https software-properties-common wget curl ca-certificates gnupg2 lsb-release systemd-journal-remote prometheus grafana opensearch opensearch-dashboards +apt-get install -y apt-transport-https software-properties-common wget curl ca-certificates gnupg2 lsb-release systemd-journal-remote prometheus grafana opensearch opensearch-dashboards nfs-common + + +# Montar servidor NFS +NFS_SERVER="ognartefactos.evlt.uma.es" +NFS_PATH="/" +LOCAL_MOUNT="/mnt" + +if ! mountpoint -q "$LOCAL_MOUNT"; then + mkdir -p "$LOCAL_MOUNT" + mount -t nfs "$NFS_SERVER:$NFS_PATH" "$LOCAL_MOUNT" +fi + + + +# Añadir dominios a /etc/hosts +HOSTNAMES=(oglog-os.mytld oglog-osdb.mytld oglog-jb.mytld oglog-jrem.mytld oglog-prom.mytld oglog-graf.mytld) + +for hostname in "${HOSTNAMES[@]}"; do + if ! grep -q "$hostname" /etc/hosts; then + echo "$OGLOG_IP $hostname" >> /etc/hosts + fi +done + + # Instalación Journalbeat y Filebeat JOURNALBEAT_URL="https://artifacts.elastic.co/downloads/beats/journalbeat/journalbeat-oss-7.12.1-amd64.deb" @@ -111,7 +139,7 @@ for file in "${files_to_copy[@]}"; do mkdir -p "$(dirname "$dest")" cp "$src" "$dest" sed -i \ - -e "s/{{IP_MAQUINA}}/$IP_MAQUINA/g" \ + -e "s/{{IP_MAQUINA}}/$OGLOG_IP/g" \ -e "s/{{OPENSEARCH_INITIAL_ADMIN_PASSWORD}}/$OPENSEARCH_INITIAL_ADMIN_PASSWORD/g" "$dest" done @@ -141,9 +169,8 @@ cp "$LOCAL_MOUNT/srv/artefactos/oglog/CA/private/oglog-jb.mytld.key.nopass.pem" cp "$LOCAL_MOUNT/srv/artefactos/oglog/CA/certs/ogagent-fb.mytld.crt.pem" /etc/filebeat/ cp "$LOCAL_MOUNT/srv/artefactos/oglog/CA/private/ogagent-fb.mytld.key.nopass.pem" /etc/filebeat/ogagent-fb.mytld.key.pem -cp CA/certs/ca.crt.pem /etc/ssl/certs/ -ln -s /etc/ssl/certs/ca.crt.pem /etc/ssl/certs/"$(openssl x509 -in /etc/ssl/certs/ca.crt.pem -hash -noout).0" - +cp "$LOCAL_MOUNT/srv/artefactos/oglog/CA/certs/ca.crt.pem" /etc/ssl/certs/ +ln -sf /etc/ssl/certs/ca.crt.pem /etc/ssl/certs/"$(openssl x509 -in /etc/ssl/certs/ca.crt.pem -hash -noout).0" # Permisos específicos chown opensearch:opensearch /etc/opensearch/* @@ -156,8 +183,9 @@ install -d -o systemd-journal-remote -g systemd-journal-remote -m 0750 /var/log/ sed -i -e '/ServerKeyFile/ s%.*%ServerKeyFile=/etc/systemd/oglog-jrem.mytld.key.pem%' /etc/systemd/journal-remote.conf sed -i -e '/ServerCertificateFile/s%.*%ServerCertificateFile=/etc/systemd/oglog-jrem.mytld.crt.pem%' /etc/systemd/journal-remote.conf sed -i -e '/TrustedCertificateFile/s%.*%TrustedCertificateFile=/etc/systemd/ca.crt.pem%' /etc/systemd/journal-remote.conf -sed -i -e '/^ARGS/s%"$% --web.config.file=/etc/prometheus/web-config.yml"%' /etc/default/prometheus - +if ! grep -q -- "--web.config.file=/etc/prometheus/web-config.yml" /etc/default/prometheus; then + sed -i -e '/^ARGS/s%"$% --web.config.file=/etc/prometheus/web-config.yml"%' /etc/default/prometheus +fi log "Descargando dashboard de Grafana..." mkdir -p /etc/grafana/dashboards @@ -185,6 +213,77 @@ for service in "${services_to_restart[@]}"; do sleep 5 done +# Añadimos la posconfiguracion una vez opensearch esta corriendo + +# Configuración de OpenSearch + +#Index pattern para filebeat + +curl -X POST "https://oglog-os.mytld:9200/.kibana/_doc/index-pattern:filebeat-*" \ + --cert /etc/journalbeat/oglog-jb.mytld.crt.pem \ + --key /etc/journalbeat/oglog-jb.mytld.key.pem \ + -u admin:CorrectHorse_BatteryStaple1 \ + -H 'Content-Type: application/json' \ + -d '{ + "type": "index-pattern", + "index-pattern": { + "title": "filebeat-*", + "timeFieldName": "@timestamp" + } + }' + +# Index pattern para Journalbeat +curl -X POST "https://oglog-os.mytld:9200/.kibana/_doc/index-pattern:journalbeat-*" \ + --cert /etc/journalbeat/oglog-jb.mytld.crt.pem \ + --key /etc/journalbeat/oglog-jb.mytld.key.pem \ + -u admin:CorrectHorse_BatteryStaple1 \ + -H 'Content-Type: application/json' \ + -d '{ + "type": "index-pattern", + "index-pattern": { + "title": "journalbeat-*", + "timeFieldName": "@timestamp" + } + }' + + +echo "Importar pipelines de ingestión de OpenSearch" +jq -c 'to_entries[]' "$base_dir/etc/opensearch/pipelines.json" | while read -r entry; do + name=$(echo "$entry" | jq -r '.key') + body=$(echo "$entry" | jq -c '.value') + + curl -X PUT "https://oglog-os.mytld:9200/_ingest/pipeline/$name" \ + --cert /etc/journalbeat/oglog-jb.mytld.crt.pem \ + --key /etc/journalbeat/oglog-jb.mytld.key.pem \ + -u "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" \ + -H "Content-Type: application/json" \ + -d "$body" +done + +echo "Importar búsquedas personalizadas de OpenSearch Dashboards" + +# Obtener los IDs reales + +JOURNALBEAT_ID=$(curl -s -X GET "https://oglog-os.mytld:9200/.kibana/_search?q=type:index-pattern" --cert /etc/journalbeat/oglog-jb.mytld.crt.pem --key /etc/journalbeat/oglog-jb.mytld.key.pem -u "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" | jq -r '.hits.hits[] | "\(.["_id"])\t\(.["_source"]["index-pattern"].title)"' | grep 'journalbeat-*' | cut -f1 | cut -d':' -f2) + +FILEBEAT_ID=$(curl -s -X GET "https://oglog-os.mytld:9200/.kibana/_search?q=type:index-pattern" --cert /etc/journalbeat/oglog-jb.mytld.crt.pem --key /etc/journalbeat/oglog-jb.mytld.key.pem -u "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" | jq -r '.hits.hits[] | "\(.["_id"])\t\(.["_source"]["index-pattern"].title)"' | grep 'filebeat-*' | cut -f1 | cut -d':' -f2) + + +# Sustituir las variables en el fichero ndjson (sin modificar el original si quieres) +cp "$base_dir/etc/opensearch-dashboards/saved_searches.ndjson" /tmp/saved_searches_modified.ndjson + +sed -i "s|__journalbeat_index__|$JOURNALBEAT_ID|g" /tmp/saved_searches_modified.ndjson +sed -i "s|__filebeat_index__|$FILEBEAT_ID|g" /tmp/saved_searches_modified.ndjson + +# Importar con overwrite +curl -X POST "https://oglog-osdb.mytld:5601/api/saved_objects/_import?overwrite=true" \ + --cert /etc/journalbeat/oglog-jb.mytld.crt.pem \ + --key /etc/journalbeat/oglog-jb.mytld.key.pem \ + -u admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD \ + -H "osd-xsrf: true" \ + -F "file=@/tmp/saved_searches_modified.ndjson" + + # Después de los reinicios log "Verificación final de servicios:" systemctl is-active journalbeat filebeat opensearch opensearch-dashboards prometheus grafana-server @@ -193,14 +292,14 @@ systemctl is-active journalbeat filebeat opensearch opensearch-dashboards promet log "Creando token para Grafana..." while IFS= read -r line; do log "$line" -done < <(./script/setup_grafana_token.sh) +done < <(./setup_grafana_token.sh) sed -i "s/__OGCORE_IP__/${OGCORE_IP}/g" ../etc/grafana/resources/datasources/datasources.json log "Importando configuracion en Grafana..." while IFS= read -r line; do log "$line" -done < <(./script/import_grafana.sh) +done < <(./import_grafana.sh) DURATION=$SECONDS