diff --git a/etc/grafana/grafana.ini b/etc/grafana/grafana.ini index 0b86e4c..7f5c8a8 100644 --- a/etc/grafana/grafana.ini +++ b/etc/grafana/grafana.ini @@ -15,3 +15,17 @@ path = /var/lib/grafana/grafana.db [auth] disable_login_form = false +#################################### SMTP / Emailing ########################## +[smtp] +enabled = true +host = smtp.qindel.es:465 +user = lgromero@qindel.com +password = *** +;cert_file = +;key_file = +skip_verify = true +from_address = lgromero@qindel.com +from_name = Grafana + +[emails] +welcome_email_on_sign_up = false diff --git a/etc/grafana/provisioning/alerting/alerts.yaml b/etc/grafana/provisioning/alerting/alerts.yaml new file mode 100644 index 0000000..810f74f --- /dev/null +++ b/etc/grafana/provisioning/alerting/alerts.yaml @@ -0,0 +1,133 @@ +apiVersion: 1 +groups: + - orgId: 1 + name: root-disk-usage-check + folder: Node Exporter Alerts + interval: 1m + rules: + - uid: deknkrm5dz400b + title: Root FS Used + condition: C + data: + - refId: A + relativeTimeRange: + from: 86400 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + adhocFilters: [] + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + exemplar: false + expr: 100 - ((node_filesystem_avail_bytes{instance="ogserver.opengnsys:9100",job="ogserver",mountpoint="/",fstype!="rootfs"} * 100) / node_filesystem_size_bytes{instance="ogserver.opengnsys:9100",job="ogserver",mountpoint="/",fstype!="rootfs"}) + format: time_series + instant: true + interval: "" + intervalFactor: 1 + intervalMs: 15000 + maxDataPoints: 43200 + range: false + refId: A + step: 240 + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 90 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + dashboardUid: rYdddlPWk + panelId: 154 + noDataState: NoData + execErrState: Error + annotations: + __dashboardUid__: rYdddlPWk + __panelId__: "154" + description: |- + Porcentaje de uso actual: ${A.Value}% + Punto de montaje: / + Job: ${__field.labels.job} + summary: 'Alerta: Uso alto del sistema de archivos raíz en ${__field.labels.instance}' + isPaused: false + notification_settings: + receiver: lgromero-email + - orgId: 1 + name: server-host-down + folder: Node Exporter Alerts + interval: 10s + rules: + - uid: deknvl4y9svlsb + title: Server Down + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + disableTextWrap: false + editorMode: builder + expr: up{job="ogserver"} == bool 0 + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + useBackend: false + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + annotations: + description: Servidor ogcore caido + summary: Servidor ogcore caido + isPaused: false + notification_settings: + receiver: lgromero-email \ No newline at end of file diff --git a/etc/grafana/provisioning/alerting/contactpoint.yaml b/etc/grafana/provisioning/alerting/contactpoint.yaml new file mode 100644 index 0000000..6878c65 --- /dev/null +++ b/etc/grafana/provisioning/alerting/contactpoint.yaml @@ -0,0 +1,19 @@ +apiVersion: 1 +contactPoints: + - orgId: 1 + name: email receiver + receivers: + - uid: "email-receiver-uid" + type: email + settings: + addresses: + disableResolveMessage: false + - orgId: 1 + name: lgromero-email + receivers: + - uid: ceknjwd00htkwb + type: email + settings: + addresses: lgromero@qindel.com;qindel@opengnsys.es + singleEmail: false + disableResolveMessage: false \ No newline at end of file diff --git a/etc/opensearch-dashboards/saved_searches.ndjson b/etc/opensearch-dashboards/saved_searches.ndjson index b3196bd..ce846f5 100644 --- a/etc/opensearch-dashboards/saved_searches.ndjson +++ b/etc/opensearch-dashboards/saved_searches.ndjson @@ -1,10 +1,10 @@ {"attributes":{"columns":["agent.name","host.ip","host.mac","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"log.file.path\",\"params\":{\"query\":\"/var/log/opengnsys.log\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"log.file.path\":\"/var/log/opengnsys.log\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Ogagent","version":1},"id":"06a268e0-d3d0-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-16T06:08:03.537Z","version":"WzQ5LDJd"} -{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"syslog.identifier\",\"params\":{\"query\":\"docker\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"docker\"}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"container.name\",\"params\":{\"query\":\"ogcore-nginx\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index\"},\"query\":{\"match_phrase\":{\"container.name\":\"ogcore-nginx\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"ogcore-nginx","version":1},"id":"0c6388d0-d3d1-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-16T06:13:48.124Z","version":"WzUzLDJd"} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"log.file.path\",\"params\":{\"query\":\"/var/log/nginx/ogcore-access.log\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"log.file.path\":\"/var/log/nginx/ogcore-access.log\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"ogcore-nginx","version":1},"id":"0c6388d0-d3d1-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-16T06:13:48.124Z","version":"WzUzLDJd"} {"attributes":{"columns":["agent.name","host.ip","host.mac","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30m\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"log.file.path\",\"params\":{\"query\":\"/var/log/opengnsys.log\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"log.file.path\":\"/var/log/opengnsys.log\"}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"agent.name\",\"params\":{\"query\":\"pc-modelo-ubuntu24-2\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index\"},\"query\":{\"match_phrase\":{\"agent.name\":\"pc-modelo-ubuntu24-2\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Ogagent - agent name","version":1},"id":"353bcbf0-d7d2-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-21T08:32:10.797Z","version":"WzU1LDJd"} {"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Kea Dhcp","version":1},"id":"39976990-d3cf-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-16T06:00:44.967Z","version":"WzQzLDJd"} {"attributes":{"columns":["parsed_message.severity","parsed_message.operation","parsed_message.http_code","parsed_message.desc","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"syslog.identifier\",\"params\":{\"query\":\"ogdhcp\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"ogdhcp\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Ogdhcp","version":1},"id":"8542fc60-d249-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-14T07:33:35.272Z","version":"WzQyLDJd"} {"attributes":{"columns":["parsed_message.severity","parsed_message.method","parsed_message.request_uri","parsed_message.operation","parsed_message.http_code","parsed_message.desc","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"syslog.identifier\",\"negate\":false,\"params\":{\"query\":\"ogboot\"},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"ogboot\"}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Ogboot","version":1},"id":"8b2a50b0-d244-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-14T07:00:24.803Z","version":"WzM1LDJd"} -{"attributes":{"columns":["parsed_json.severity","parsed_json.component","parsed_json.operation","parsed_json.desc"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"syslog.identifier\",\"params\":{\"query\":\"docker\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"docker\"}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"container.name\",\"params\":{\"query\":\"ogcore-php\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index\"},\"query\":{\"match_phrase\":{\"container.name\":\"ogcore-php\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"ogcore-php","version":1},"id":"abe87370-d188-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-13T08:30:40.160Z","version":"WzksMl0="} +{"attributes":{"columns":["parsed_json.severity","parsed_json.component","parsed_json.operation","parsed_json.desc"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"syslog.identifier\",\"params\":{\"query\":\"ogcore\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"ogcore\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"ogcore-php","version":1},"id":"abe87370-d188-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-13T08:30:40.160Z","version":"WzksMl0="} {"exportedCount":7,"missingRefCount":0,"missingReferences":[]} diff --git a/script/import_grafana.sh b/script/import_grafana.sh index 7d93a38..d81c802 100755 --- a/script/import_grafana.sh +++ b/script/import_grafana.sh @@ -39,16 +39,16 @@ for f in "$RESOURCE_DIR/dashboards"/*.json; do done echo "Dashboards importados." # Importar alertas -if [ -f "$RESOURCE_DIR/alerts/alert-rules.json" ]; then - echo "Importando alert rules..." - jq -c '.[]' "$RESOURCE_DIR/alerts/alert-rules.json" | while read -r alert; do - curl -s -X POST "$GRAFANA_URL/api/v1/provisioning/alert-rules" \ - -H "Authorization: Bearer $API_TOKEN" \ - -H "Content-Type: application/json" \ - -d "$alert" - done - echo "Alertas importadas." -fi +# if [ -f "$RESOURCE_DIR/alerts/alert-rules.json" ]; then +# echo "Importando alert rules..." +# jq -c '.[]' "$RESOURCE_DIR/alerts/alert-rules.json" | while read -r alert; do +# curl -s -X POST "$GRAFANA_URL/api/v1/provisioning/alert-rules" \ +# -H "Authorization: Bearer $API_TOKEN" \ +# -H "Content-Type: application/json" \ +# -d "$alert" +# done +# echo "Alertas importadas." +# fi echo "Importación completa." diff --git a/script/oglog_installer.sh b/script/oglog_installer.sh index 6aa711f..d4d9274 100755 --- a/script/oglog_installer.sh +++ b/script/oglog_installer.sh @@ -104,6 +104,8 @@ for hostname in "${HOSTNAMES[@]}"; do fi done +echo "ogserver.opengnsys $OGCORE_IP" >> /etc/hosts + # Instalación Journalbeat y Filebeat @@ -132,6 +134,8 @@ files_to_copy=( "grafana/grafana.ini" "grafana/provisioning/datasources/prometheus.yaml" "grafana/provisioning/dashboards/dashboard.yaml" + "grafana/provisioning/alerting/alerts.yaml" + "grafana/provisioning/alerting/contactpoint.yaml" ) for file in "${files_to_copy[@]}"; do