refs #1666 adds new alarms configuration, updates opensearch dashboard saved search without docker

etc/grafana/grafana.ini

@@ -15,3 +15,17 @@ path = /var/lib/grafana/grafana.db
 [auth]
 disable_login_form = false
 
+#################################### SMTP / Emailing ##########################
+[smtp]
+enabled = true
+host = smtp.qindel.es:465
+user = lgromero@qindel.com
+password = ***
+;cert_file =
+;key_file =
+skip_verify = true
+from_address = lgromero@qindel.com
+from_name = Grafana
+
+[emails]
+welcome_email_on_sign_up = false

etc/grafana/provisioning/alerting/alerts.yaml

@@ -0,0 +1,133 @@
+apiVersion: 1
+groups:
+    - orgId: 1
+      name: root-disk-usage-check
+      folder: Node Exporter Alerts
+      interval: 1m
+      rules:
+        - uid: deknkrm5dz400b
+          title: Root FS Used
+          condition: C
+          data:
+            - refId: A
+              relativeTimeRange:
+                from: 86400
+                to: 0
+              datasourceUid: PBFA97CFB590B2093
+              model:
+                adhocFilters: []
+                datasource:
+                    type: prometheus
+                    uid: PBFA97CFB590B2093
+                editorMode: code
+                exemplar: false
+                expr: 100 - ((node_filesystem_avail_bytes{instance="ogserver.opengnsys:9100",job="ogserver",mountpoint="/",fstype!="rootfs"} * 100) / node_filesystem_size_bytes{instance="ogserver.opengnsys:9100",job="ogserver",mountpoint="/",fstype!="rootfs"})
+                format: time_series
+                instant: true
+                interval: ""
+                intervalFactor: 1
+                intervalMs: 15000
+                maxDataPoints: 43200
+                range: false
+                refId: A
+                step: 240
+            - refId: C
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 90
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params:
+                            - C
+                      reducer:
+                        params: []
+                        type: last
+                      type: query
+                datasource:
+                    type: __expr__
+                    uid: __expr__
+                expression: A
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: C
+                type: threshold
+          dashboardUid: rYdddlPWk
+          panelId: 154
+          noDataState: NoData
+          execErrState: Error
+          annotations:
+            __dashboardUid__: rYdddlPWk
+            __panelId__: "154"
+            description: |-
+                Porcentaje de uso actual: ${A.Value}%
+                Punto de montaje: /
+                Job: ${__field.labels.job}
+            summary: 'Alerta: Uso alto del sistema de archivos raíz en ${__field.labels.instance}'
+          isPaused: false
+          notification_settings:
+            receiver: lgromero-email
+    - orgId: 1
+      name: server-host-down
+      folder: Node Exporter Alerts
+      interval: 10s
+      rules:
+        - uid: deknvl4y9svlsb
+          title: Server Down
+          condition: C
+          data:
+            - refId: A
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: PBFA97CFB590B2093
+              model:
+                disableTextWrap: false
+                editorMode: builder
+                expr: up{job="ogserver"} == bool 0
+                fullMetaSearch: false
+                includeNullMetadata: true
+                instant: true
+                intervalMs: 1000
+                legendFormat: __auto
+                maxDataPoints: 43200
+                range: false
+                refId: A
+                useBackend: false
+            - refId: C
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 1
+                        type: lt
+                      operator:
+                        type: and
+                      query:
+                        params:
+                            - C
+                      reducer:
+                        params: []
+                        type: last
+                      type: query
+                datasource:
+                    type: __expr__
+                    uid: __expr__
+                expression: A
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: C
+                type: threshold
+          noDataState: NoData
+          execErrState: Error
+          annotations:
+            description: Servidor ogcore caido
+            summary: Servidor ogcore caido
+          isPaused: false
+          notification_settings:
+            receiver: lgromero-email

etc/grafana/provisioning/alerting/contactpoint.yaml

@@ -0,0 +1,19 @@
+apiVersion: 1
+contactPoints:
+    - orgId: 1
+      name: email receiver
+      receivers:
+        - uid: "email-receiver-uid"
+          type: email
+          settings:
+            addresses: <example@email.com>
+          disableResolveMessage: false
+    - orgId: 1
+      name: lgromero-email
+      receivers:
+        - uid: ceknjwd00htkwb
+          type: email
+          settings:
+            addresses: lgromero@qindel.com;qindel@opengnsys.es
+            singleEmail: false
+          disableResolveMessage: false

etc/opensearch-dashboards/saved_searches.ndjson

@@ -1,10 +1,10 @@
 {"attributes":{"columns":["agent.name","host.ip","host.mac","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"log.file.path\",\"params\":{\"query\":\"/var/log/opengnsys.log\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"log.file.path\":\"/var/log/opengnsys.log\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Ogagent","version":1},"id":"06a268e0-d3d0-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-16T06:08:03.537Z","version":"WzQ5LDJd"}
-{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"syslog.identifier\",\"params\":{\"query\":\"docker\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"docker\"}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"container.name\",\"params\":{\"query\":\"ogcore-nginx\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index\"},\"query\":{\"match_phrase\":{\"container.name\":\"ogcore-nginx\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"ogcore-nginx","version":1},"id":"0c6388d0-d3d1-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-16T06:13:48.124Z","version":"WzUzLDJd"}
+{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"log.file.path\",\"params\":{\"query\":\"/var/log/nginx/ogcore-access.log\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"log.file.path\":\"/var/log/nginx/ogcore-access.log\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"ogcore-nginx","version":1},"id":"0c6388d0-d3d1-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-16T06:13:48.124Z","version":"WzUzLDJd"}
 {"attributes":{"columns":["agent.name","host.ip","host.mac","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30m\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"log.file.path\",\"params\":{\"query\":\"/var/log/opengnsys.log\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"log.file.path\":\"/var/log/opengnsys.log\"}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"agent.name\",\"params\":{\"query\":\"pc-modelo-ubuntu24-2\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index\"},\"query\":{\"match_phrase\":{\"agent.name\":\"pc-modelo-ubuntu24-2\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Ogagent - agent name","version":1},"id":"353bcbf0-d7d2-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-21T08:32:10.797Z","version":"WzU1LDJd"}
 {"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Kea Dhcp","version":1},"id":"39976990-d3cf-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__filebeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-16T06:00:44.967Z","version":"WzQzLDJd"}
 {"attributes":{"columns":["parsed_message.severity","parsed_message.operation","parsed_message.http_code","parsed_message.desc","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"syslog.identifier\",\"params\":{\"query\":\"ogdhcp\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"ogdhcp\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Ogdhcp","version":1},"id":"8542fc60-d249-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-14T07:33:35.272Z","version":"WzQyLDJd"}
 {"attributes":{"columns":["parsed_message.severity","parsed_message.method","parsed_message.request_uri","parsed_message.operation","parsed_message.http_code","parsed_message.desc","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"syslog.identifier\",\"negate\":false,\"params\":{\"query\":\"ogboot\"},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"ogboot\"}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Ogboot","version":1},"id":"8b2a50b0-d244-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-14T07:00:24.803Z","version":"WzM1LDJd"}
-{"attributes":{"columns":["parsed_json.severity","parsed_json.component","parsed_json.operation","parsed_json.desc"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"syslog.identifier\",\"params\":{\"query\":\"docker\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"docker\"}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"container.name\",\"params\":{\"query\":\"ogcore-php\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index\"},\"query\":{\"match_phrase\":{\"container.name\":\"ogcore-php\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"ogcore-php","version":1},"id":"abe87370-d188-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-13T08:30:40.160Z","version":"WzksMl0="}
+{"attributes":{"columns":["parsed_json.severity","parsed_json.component","parsed_json.operation","parsed_json.desc"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"Europe/Madrid\",\"min_doc_count\":1}}},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"syslog.identifier\",\"params\":{\"query\":\"ogcore\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"syslog.identifier\":\"ogcore\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"ogcore-php","version":1},"id":"abe87370-d188-11ef-9b0b-2d28387787cc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"__journalbeat_index__","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2025-01-13T08:30:40.160Z","version":"WzksMl0="}
 {"exportedCount":7,"missingRefCount":0,"missingReferences":[]}
 
 

script/import_grafana.sh

@@ -39,16 +39,16 @@ for f in "$RESOURCE_DIR/dashboards"/*.json; do
 done
 echo "Dashboards importados."
 # Importar alertas
-if [ -f "$RESOURCE_DIR/alerts/alert-rules.json" ]; then
-  echo "Importando alert rules..."
-  jq -c '.[]' "$RESOURCE_DIR/alerts/alert-rules.json" | while read -r alert; do
-    curl -s -X POST "$GRAFANA_URL/api/v1/provisioning/alert-rules" \
-      -H "Authorization: Bearer $API_TOKEN" \
-      -H "Content-Type: application/json" \
-      -d "$alert"
-  done
-  echo "Alertas importadas."
-fi
+# if [ -f "$RESOURCE_DIR/alerts/alert-rules.json" ]; then
+#   echo "Importando alert rules..."
+#   jq -c '.[]' "$RESOURCE_DIR/alerts/alert-rules.json" | while read -r alert; do
+#     curl -s -X POST "$GRAFANA_URL/api/v1/provisioning/alert-rules" \
+#       -H "Authorization: Bearer $API_TOKEN" \
+#       -H "Content-Type: application/json" \
+#       -d "$alert"
+#   done
+#   echo "Alertas importadas."
+# fi
 
 echo "Importación completa."
 

script/oglog_installer.sh

@@ -104,6 +104,8 @@ for hostname in "${HOSTNAMES[@]}"; do
   fi
 done
 
+echo "ogserver.opengnsys $OGCORE_IP" >> /etc/hosts
+
 
 
 # Instalación Journalbeat y Filebeat
@@ -132,6 +134,8 @@ files_to_copy=(
   "grafana/grafana.ini"
   "grafana/provisioning/datasources/prometheus.yaml"
   "grafana/provisioning/dashboards/dashboard.yaml"
+  "grafana/provisioning/alerting/alerts.yaml"
+  "grafana/provisioning/alerting/contactpoint.yaml"
 )
 
 for file in "${files_to_copy[@]}"; do