From 40c0b91644b2cf90dcd78f8e71027eb13f1d41a9 Mon Sep 17 00:00:00 2001
From: root <dpilar>
Date: Tue, 14 Jan 2025 09:53:48 +0000
Subject: [PATCH] refs #1299

---
 script/agent.sh               |  92 ++++++++++++++++++++++++++
 script/pipeline-bulk.sh       |  14 ++++
 script/pipeline-individual.sh |  35 ++++++++++
 script/pipeline.sh            |  20 ++++++
 script/script.sh              | 117 ++++++++++++++++++++--------------
 5 files changed, 231 insertions(+), 47 deletions(-)
 create mode 100755 script/agent.sh
 create mode 100755 script/pipeline-bulk.sh
 create mode 100755 script/pipeline-individual.sh
 create mode 100755 script/pipeline.sh

diff --git a/script/agent.sh b/script/agent.sh
new file mode 100755
index 0000000..1b04d19
--- /dev/null
+++ b/script/agent.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+set -e  # Detener el script si ocurre un error
+
+# Configurar Filebeat
+echo "Verificando conectividad"
+curl -I --connect-timeout 10 --max-time 30 -s -o /dev/null --retry 5 https://artifacts.elastic.co/downloads/beats/filebeat/
+
+if [[ $? -ne 0 ]]; then
+  echo "ERROR: No se puede conectar a https://artifacts.elastic.co/downloads/beats/filebeat. Verifica tu conexión a Internet o la disponibilidad"
+  exit 1
+fi
+
+curl --connect-timeout 10 --max-time 60 --retry 5 -L -o /tmp/filebeat-oss-7.12.1-amd64.deb https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-oss-7.12.1-amd64.deb
+sudo dpkg -i /tmp/filebeat-oss-7.12.1-amd64.deb
+cp CA/certs/ogagent-fb.mytld.crt.pem          /etc/filebeat/
+cp CA/private/ogagent-fb.mytld.key.nopass.pem /etc/filebeat/ogagent-fb.mytld.key.pem
+
+cat >/etc/filebeat/filebeat.yml <<EOF
+filebeat.inputs:
+- type: log
+  enabled: true
+  paths:
+    - /var/log/opengnsys.log
+    - /home/*/opengnsys.log
+
+setup.template.settings:
+  index.number_of_shards: 1
+
+output.elasticsearch:
+  hosts: ["oglog-os.mytld:9200"]
+  username: "admin"
+  password: "\$OPENSEARCH_INITIAL_ADMIN_PASSWORD"
+  protocol: "https"
+  ssl.enabled: true
+  ssl.verification_mode: full
+  ssl.certificate: "/etc/filebeat/ogagent-fb.mytld.crt.pem"
+  ssl.key: "/etc/filebeat/ogagent-fb.mytld.key.pem"
+
+processors:
+  - add_host_metadata:
+      when.not.contains.tags: forwarded
+  - add_cloud_metadata: ~
+  - add_docker_metadata: ~
+  - add_kubernetes_metadata: ~
+
+seccomp.enabled: false
+EOF
+
+systemctl enable --now filebeat
+
+# Verificar que la variable de entorno IP_SERVER esté configurada
+if [[ -z "$IP_SERVER" ]]; then
+  echo "ERROR: La variable de entorno IP_SERVER no está configurada."
+  echo "Por favor, exporta IP_SERVER antes de ejecutar este script."
+  exit 1
+fi
+
+# Actualizar /etc/hosts con los nombres de dominio
+cat >>/etc/hosts <<EOF
+$IP_SERVER oglog-jrem.mytld
+EOF
+
+echo "Actualizando paquetes e instalando dependencias..."
+apt-get update
+apt-get -y install \
+  prometheus-node-exporter \
+  systemd-journal-remote
+
+echo "Configurando TLS y copiando certificados..."
+cp CA/certs/ca.crt.pem /etc/ssl/certs/
+ln -sf /etc/ssl/certs/ca.crt.pem /etc/ssl/certs/$(openssl x509 -in /etc/ssl/certs/ca.crt.pem -hash -noout).0
+cp CA/certs/ogserver.mytld.crt.pem /etc/ssl/certs/
+cp CA/private/ogserver.mytld.key.nopass.pem /etc/ssl/private/ogserver.mytld.key.pem
+chmod 600 /etc/ssl/private/ogserver.mytld.key.pem
+chown root:root /etc/ssl/private/ogserver.mytld.key.pem
+
+echo "Configurando systemd-journal-upload..."
+sed -i -e '/DynamicUser/s/.*/DynamicUser=no/' /usr/lib/systemd/system/systemd-journal-upload.service
+sed -i -e '/User/       s/.*/User=root/'      /usr/lib/systemd/system/systemd-journal-upload.service
+systemctl daemon-reload
+
+sed -i -e '/URL/                   s%.*%URL=https://oglog-jrem.mytld:19532%'                          /etc/systemd/journal-upload.conf
+sed -i -e '/ServerKeyFile/         s%.*%ServerKeyFile=/etc/ssl/private/ogserver.mytld.key.pem%'       /etc/systemd/journal-upload.conf
+sed -i -e '/ServerCertificateFile/ s%.*%ServerCertificateFile=/etc/ssl/certs/ogserver.mytld.crt.pem%' /etc/systemd/journal-upload.conf
+sed -i -e '/TrustedCertificateFile/s%.*%TrustedCertificateFile=/etc/ssl/certs/ca.crt.pem%'            /etc/systemd/journal-upload.conf
+
+echo "Habilitando y arrancando systemd-journal-upload..."
+systemctl enable --now systemd-journal-upload
+
+systemctl status systemd-journal-upload --no-pager
+
+echo "Configuración completada con éxito. Los logs se están enviando al servidor remoto."
diff --git a/script/pipeline-bulk.sh b/script/pipeline-bulk.sh
new file mode 100755
index 0000000..e22dcfe
--- /dev/null
+++ b/script/pipeline-bulk.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Ejecutar todos los pipelines definidos en pipeline.yml
+if [[ -f "pipeline.yml" ]]; then
+  echo "Ejecutando pipelines definidos en pipeline.yml..."
+  # Leer el archivo completo y enviar su contenido a OpenSearch
+  curl -X PUT "https://localhost:9200/_ingest/pipeline/_bulk" \
+  -H "Content-Type: application/json" \
+  -u "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" \
+  --cacert "CA/certs/ca.crt.pem" \
+  -d @pipeline.yml
+else
+  echo "No se encontró pipeline.yml. No se ejecutaron pipelines."
+fi
diff --git a/script/pipeline-individual.sh b/script/pipeline-individual.sh
new file mode 100755
index 0000000..5233ab4
--- /dev/null
+++ b/script/pipeline-individual.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Verificar si existe el archivo pipeline.yml
+if [[ -f "pipeline.yml" ]]; then
+  echo "Analizando y ejecutando pipelines definidos en pipeline.yml..."
+  
+  # Dividir el archivo en partes individuales por cada pipeline
+  awk '/^---/ {close("pipeline.tmp"); filename="pipeline_" NR ".tmp"} {print > filename}' pipeline.yml
+
+  # Procesar cada archivo temporal creado por awk
+  for pipeline_file in pipeline_*.tmp; do
+    if [[ -f "$pipeline_file" ]]; then
+      # Extraer el nombre del pipeline del archivo
+      pipeline_name=$(awk '/^name:/ {print $2; exit}' "$pipeline_file")
+      
+      if [[ -n "$pipeline_name" ]]; then
+        echo "Ejecutando pipeline: $pipeline_name..."
+        curl -X PUT "https://localhost:9200/_ingest/pipeline/$pipeline_name" \
+        -H "Content-Type: application/json" \
+        -u "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" \
+        --cacert "CA/certs/ca.crt.pem" \
+        -d @"$pipeline_file"
+      else
+        echo "ERROR: No se pudo extraer el nombre del pipeline de $pipeline_file."
+      fi
+
+      # Eliminar archivo temporal después de procesarlo
+      rm "$pipeline_file"
+    else
+      echo "ERROR: No se encontró el archivo temporal $pipeline_file."
+    fi
+  done
+else
+  echo "No se encontró pipeline.yml. No se ejecutaron pipelines."
+fi
diff --git a/script/pipeline.sh b/script/pipeline.sh
new file mode 100755
index 0000000..e5bc4f5
--- /dev/null
+++ b/script/pipeline.sh
@@ -0,0 +1,20 @@
+# Ejecutar pipelines definidos en pipeline.yml
+if [[ -f "pipeline.yml" ]]; then
+  echo "Ejecutando pipelines definidos en pipeline.yml..."
+  while IFS= read -r pipeline; do
+    if [[ -n "$pipeline" ]]; then
+      pipeline_file="${pipeline}.yml"
+      if [[ -f "$pipeline_file" ]]; then
+        curl -X PUT "https://localhost:9200/_ingest/pipeline/$pipeline" \
+        -H "Content-Type: application/json" \
+        -u "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" \
+        --cacert "CA/certs/ca.crt.pem" \
+        -d @"$pipeline_file"
+      else
+        echo "ERROR: No se encontró el archivo $pipeline_file."
+      fi
+    fi
+  done < <(grep -o '^[^#]*' "pipeline.yml" | grep -v '^$')
+else
+  echo "No se encontró pipeline.yml. No se ejecutaron pipelines."
+fi
diff --git a/script/script.sh b/script/script.sh
index 9df980f..186e938 100755
--- a/script/script.sh
+++ b/script/script.sh
@@ -64,6 +64,7 @@ setup.template.settings:
 output.elasticsearch:
   hosts: ["oglog-os.mytld:9200"]
   username: "admin"
+  pipeline: "simple_parse_pipeline"
   password: "$OPENSEARCH_INITIAL_ADMIN_PASSWORD"
   protocol: "https"
   ssl.enabled: true
@@ -79,53 +80,6 @@ EOF
 
 systemctl enable --now journalbeat
 
-# Configurar Filebeat
-
-# Prueba de conexión a la URL de la clave GPG
-echo "Verificando conectividad"
-curl -I --connect-timeout 10 --max-time 30 -s -o /dev/null --retry 5 https://artifacts.elastic.co/downloads/beats/filebeat/
-
-if [[ $? -ne 0 ]]; then
-  echo "ERROR: No se puede conectar a https://artifacts.elastic.co/downloads/beats/filebeat. Verifica tu conexión a Internet o la disponibilidad"
-  exit 1
-fi
-
-curl --connect-timeout 10 --max-time 60 --retry 5 -L -o /tmp/filebeat-oss-7.12.1-amd64.deb https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-oss-7.12.1-amd64.deb
-sudo dpkg -i /tmp/filebeat-oss-7.12.1-amd64.deb
-cp CA/certs/ogagent-fb.mytld.crt.pem          /etc/filebeat/
-cp CA/private/ogagent-fb.mytld.key.nopass.pem /etc/filebeat/ogagent-fb.mytld.key.pem
-cat >/etc/filebeat/filebeat.yml <<EOF
-filebeat.inputs:
-- type: log
-  enabled: true
-  paths:
-    - /var/log/opengnsys.log
-    - /home/*/opengnsys.log
-
-setup.template.settings:
-  index.number_of_shards: 1
-
-output.elasticsearch:
-  hosts: ["oglog-os.mytld:9200"]
-  username: "admin"
-  password: "$OPENSEARCH_INITIAL_ADMIN_PASSWORD"
-  protocol: "https"
-  ssl.enabled: true
-  ssl.verification_mode: full
-  ssl.certificate: "/etc/filebeat/ogagent-fb.mytld.crt.pem"
-  ssl.key: "/etc/filebeat/ogagent-fb.mytld.key.pem"
-
-processors:
-  - add_host_metadata:
-      when.not.contains.tags: forwarded
-  - add_cloud_metadata: ~
-  - add_docker_metadata: ~
-  - add_kubernetes_metadata: ~
-
-seccomp.enabled: false
-EOF
-systemctl enable --now filebeat
-
 # Configurar repositorios y llaves para OpenSearch
 curl -o- https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --dearmor --batch --yes -o /usr/share/keyrings/opensearch-keyring
 echo "deb [signed-by=/usr/share/keyrings/opensearch-keyring] https://artifacts.opensearch.org/releases/bundle/opensearch/2.x/apt stable main" > /etc/apt/sources.list.d/opensearch-2.x.list
@@ -176,6 +130,75 @@ EOF
 # Habilitar servicios de OpenSearch
 systemctl enable --now opensearch.service opensearch-dashboards.service
 
+# Esperar a que OpenSearch esté disponible
+echo "Esperando a que OpenSearch esté disponible..."
+until curl -s --fail \
+    --cert /etc/opensearch/oglog-os.mytld.crt.pem \
+    --key /etc/opensearch/oglog-os.mytld.key.pem \
+    --cacert /etc/opensearch/ca.crt.pem \
+    -u "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" \
+    "https://oglog-os.mytld:9200/_cluster/health"; do
+    sleep 5
+done
+echo "OpenSearch está disponible."
+
+# Configurar pipeline por defecto
+curl -XPUT "https://oglog-os.mytld:9200/_ingest/pipeline/simple_parse_pipeline" \
+    --cert /etc/opensearch/oglog-os.mytld.crt.pem \
+    --key /etc/opensearch/oglog-os.mytld.key.pem \
+    --cacert /etc/opensearch/ca.crt.pem \
+    -u "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" \
+    -H 'Content-Type: application/json' \
+    -d'
+{
+  "description": "Parse logs to extract http_code and desc, supporting various severity levels",
+  "processors": [
+    {
+      "script": {
+        "if": "ctx.syslog?.identifier != '\''ogboot'\''",
+        "source": "ctx.debug = '\''Skipped: identifier is '\'' + (ctx.syslog?.identifier ?: '\''undefined'\''); ctx.pipeline_stop = true;"
+      }
+    },
+    {
+      "set": {
+        "field": "debug",
+        "value": "Processed: identifier is ogboot"
+      }
+    },
+    {
+      "gsub": {
+        "field": "message",
+        "pattern": "^app\\.[A-Z]+: ",
+        "replacement": "",
+        "ignore_failure": true
+      }
+    },
+    {
+      "json": {
+        "field": "message",
+        "target_field": "parsed_message",
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "http_code",
+        "value": "{{parsed_message.http_code}}",
+        "ignore_empty_value": true
+      }
+    },
+    {
+      "set": {
+        "field": "description",
+        "value": "{{parsed_message.desc}}",
+        "ignore_empty_value": true
+      }
+    }
+  ]
+}'
+echo "Pipeline simple_parse_pipeline configurado."
+
+
 # Configurar systemd-journal-remote
 cp CA/certs/ca.crt.pem /etc/systemd/
 cp CA/certs/oglog-jrem.mytld.crt.pem /etc/systemd/