Setup filebeat and metricbeat

filebeat/filebeat.yml

@@ -0,0 +1,261 @@
+###################### Filebeat Configuration Example #########################
+
+# This file is an example configuration file highlighting only the most common
+# options. The filebeat.reference.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/filebeat/index.html
+
+# For more available modules and options, please see the filebeat.reference.yml sample
+# configuration file.
+
+# ============================== Filebeat inputs ===============================
+
+filebeat.inputs:
+
+# Each - is an input. Most options can be set at the input level, so
+# you can use different inputs for various configurations.
+# Below are the input-specific configurations.
+
+# filestream is an input for collecting log messages from files.
+- type: filestream
+
+  # Unique ID among all inputs, an ID is required.
+  id: my-filestream-id
+
+  # Change to true to enable this input configuration.
+  enabled: false
+
+  # Paths that should be crawled and fetched. Glob based paths.
+  paths:
+    - /var/log/*.log
+    #- c:\programdata\elasticsearch\logs\*
+
+  # Exclude lines. A list of regular expressions to match. It drops the lines that are
+  # matching any regular expression from the list.
+  # Line filtering happens after the parsers pipeline. If you would like to filter lines
+  # before parsers, use include_message parser.
+  #exclude_lines: ['^DBG']
+
+  # Include lines. A list of regular expressions to match. It exports the lines that are
+  # matching any regular expression from the list.
+  # Line filtering happens after the parsers pipeline. If you would like to filter lines
+  # before parsers, use include_message parser.
+  #include_lines: ['^ERR', '^WARN']
+
+  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
+  # are matching any regular expression from the list. By default, no files are dropped.
+  #prospector.scanner.exclude_files: ['.gz$']
+
+  # Optional additional fields. These fields can be freely picked
+  # to add additional information to the crawled log files for filtering
+  #fields:
+  #  level: debug
+  #  review: 1
+
+# journald is an input for collecting logs from Journald
+#- type: journald
+
+  # Unique ID among all inputs, if the ID changes, all entries
+  # will be re-ingested
+  #id: my-journald-id
+
+  # The position to start reading from the journal, valid options are:
+  #  - head: Starts reading at the beginning of the journal.
+  #  - tail: Starts reading at the end of the journal.
+  #    This means that no events will be sent until a new message is written.
+  #  - since: Use also the `since` option to determine when to start reading from.
+  #seek: head
+
+  # A time offset from the current time to start reading from.
+  # To use since, seek option must be set to since.
+  #since: -24h
+
+  # Collect events from the service and messages about the service,
+  # including coredumps.
+  #units:
+    #- docker.service
+
+# ============================== Filebeat modules ==============================
+
+filebeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  #reload.period: 10s
+
+# ======================= Elasticsearch template setting =======================
+setup.ilm.enabled: false
+setup.template.enabled: false
+setup.template.settings:
+  index.number_of_shards: 1
+  #index.codec: best_compression
+  #_source.enabled: false
+
+
+# ================================== General ===================================
+
+# The name of the shipper that publishes the network data. It can be used to group
+# all the transactions sent by a single shipper in the web interface.
+#name:
+
+# The tags of the shipper are included in their field with each
+# transaction published.
+#tags: ["service-X", "web-tier"]
+
+# Optional fields that you can specify to add additional information to the
+# output.
+#fields:
+#  env: staging
+
+# ================================= Dashboards =================================
+# These settings control loading the sample dashboards to the Kibana index. Loading
+# the dashboards is disabled by default and can be enabled either by setting the
+# options here or by using the `setup` command.
+#setup.dashboards.enabled: false
+
+# The URL from where to download the dashboard archive. By default, this URL
+# has a value that is computed based on the Beat name and version. For released
+# versions, this URL points to the dashboard archive on the artifacts.elastic.co
+# website.
+#setup.dashboards.url:
+
+# =================================== Kibana ===================================
+
+# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
+# This requires a Kibana endpoint configuration.
+setup.kibana:
+
+  # Kibana Host
+  # Scheme and port can be left out and will be set to the default (http and 5601)
+  # In case you specify and additional path, the scheme is required: http://localhost:5601/path
+  # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
+  #host: "localhost:5601"
+
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
+# =============================== Elastic Cloud ================================
+
+# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
+# ================================== Outputs ===================================
+
+# Configure what output to use when sending the data collected by the beat.
+
+# ---------------------------- Elasticsearch Output ----------------------------
+output.elasticsearch:
+  # Array of hosts to connect to.
+  hosts: ["localhost:9200"]
+
+  # Performance preset - one of "balanced", "throughput", "scale",
+  # "latency", or "custom".
+  preset: balanced
+  username: "replace-user"
+  password: "replace-password"
+  # Protocol - either `http` (default) or `https`.
+  #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
+  #username: "elastic"
+  #password: "changeme"
+
+# ------------------------------ Logstash Output -------------------------------
+#output.logstash:
+  # The Logstash hosts
+  #hosts: ["localhost:5044"]
+
+  # Optional SSL. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+# ================================= Processors =================================
+processors:
+  - add_host_metadata:
+      when.not.contains.tags: forwarded
+  - add_cloud_metadata: ~
+  - add_docker_metadata: ~
+  - add_kubernetes_metadata: ~
+
+# ================================== Logging ===================================
+
+# Sets log level. The default log level is info.
+# Available log levels are: error, warning, info, debug
+#logging.level: debug
+
+# At debug level, you can selectively enable logging only for some components.
+# To enable all selectors, use ["*"]. Examples of other selectors are "beat",
+# "publisher", "service".
+#logging.selectors: ["*"]
+
+# ============================= X-Pack Monitoring ==============================
+# Filebeat can export internal metrics to a central Elasticsearch monitoring
+# cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The
+# reporting is disabled by default.
+
+# Set to true to enable the monitoring reporter.
+#monitoring.enabled: false
+
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Filebeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
+# Uncomment to send the metrics to Elasticsearch. Most settings from the
+# Elasticsearch outputs are accepted here as well.
+# Note that the settings should point to your Elasticsearch *monitoring* cluster.
+# Any setting that is not set is automatically inherited from the Elasticsearch
+# output configuration, so if you have the Elasticsearch output configured such
+# that it is pointing to your Elasticsearch monitoring cluster, you can simply
+# uncomment the following line.
+#monitoring.elasticsearch:
+
+# ============================== Instrumentation ===============================
+
+# Instrumentation support for the filebeat.
+#instrumentation:
+    # Set to true to enable instrumentation of filebeat.
+    #enabled: false
+
+    # Environment in which filebeat is running on (eg: staging, production, etc.)
+    #environment: ""
+
+    # APM Server hosts to report instrumentation results to.
+    #hosts:
+    #  - http://localhost:8200
+
+    # API Key for the APM Server(s).
+    # If api_key is set then secret_token will be ignored.
+    #api_key:
+
+    # Secret token for the APM Server(s).
+    #secret_token:
+
+
+# ================================= Migration ==================================
+
+# This allows to enable 6.7 migration aliases
+#migration.6_to_7.enabled: true

filebeat/modules.d/system.yml

@@ -0,0 +1,19 @@
+# Module: system
+# Docs: https://www.elastic.co/guide/en/beats/filebeat/main/filebeat-module-system.html
+
+- module: system
+  # Syslog
+  syslog:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Authorization logs
+  auth:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

logstash/conf.d/90-output.conf

@@ -2,7 +2,6 @@ output {
   # Forward logs to Elasticsearch
   elasticsearch {
     hosts => ["http://localhost:9200"]
-    index => "replace-index"  # Index pattern based on log type and date
     user => "replace-user"
     password => "replace-password"
   }

logstash/logstash.yml

@@ -0,0 +1,396 @@
+# Settings file in YAML
+#
+# Settings can be specified either in hierarchical form, e.g.:
+#
+#   pipeline:
+#     batch:
+#       size: 125
+#       delay: 5
+#
+# Or as flat keys:
+#
+#   pipeline.batch.size: 125
+#   pipeline.batch.delay: 5
+#
+# ------------  Node identity ------------
+#
+# Use a descriptive name for the node:
+#
+# node.name: test
+#
+# If omitted the node name will default to the machine's host name
+#
+# ------------ Data path ------------------
+#
+# Which directory should be used by logstash and its plugins
+# for any persistent needs. Defaults to LOGSTASH_HOME/data
+#
+path.data: /var/lib/logstash
+#
+# ------------ Pipeline Settings --------------
+#
+# The ID of the pipeline.
+#
+# pipeline.id: main
+#
+# Set the number of workers that will, in parallel, execute the filters+outputs
+# stage of the pipeline.
+#
+# This defaults to the number of the host's CPU cores.
+#
+# pipeline.workers: 2
+#
+# How many events to retrieve from inputs before sending to filters+workers
+#
+# pipeline.batch.size: 125
+#
+# How long to wait in milliseconds while polling for the next event
+# before dispatching an undersized batch to filters+outputs
+#
+# pipeline.batch.delay: 50
+#
+# Force Logstash to exit during shutdown even if there are still inflight
+# events in memory. By default, logstash will refuse to quit until all
+# received events have been pushed to the outputs.
+#
+# WARNING: Enabling this can lead to data loss during shutdown
+#
+# pipeline.unsafe_shutdown: false
+#
+# Set the pipeline event ordering. Options are "auto" (the default), "true" or "false".
+# "auto" automatically enables ordering if the 'pipeline.workers' setting
+# is also set to '1', and disables otherwise.
+# "true" enforces ordering on the pipeline and prevent logstash from starting
+# if there are multiple workers.
+# "false" disables any extra processing necessary for preserving ordering.
+#
+# pipeline.ordered: auto
+#
+# Sets the pipeline's default value for `ecs_compatibility`, a setting that is
+# available to plugins that implement an ECS Compatibility mode for use with
+# the Elastic Common Schema.
+# Possible values are:
+# - disabled
+# - v1
+# - v8 (default)
+# Pipelines defined before Logstash 8 operated without ECS in mind. To ensure a
+# migrated pipeline continues to operate as it did before your upgrade, opt-OUT
+# of ECS for the individual pipeline in its `pipelines.yml` definition. Setting
+# it here will set the default for _all_ pipelines, including new ones.
+#
+# pipeline.ecs_compatibility: v8
+#
+# ------------ Pipeline Configuration Settings --------------
+#
+# Where to fetch the pipeline configuration for the main pipeline
+#
+# path.config:
+#
+# Pipeline configuration string for the main pipeline
+#
+# config.string:
+#
+# At startup, test if the configuration is valid and exit (dry run)
+#
+# config.test_and_exit: false
+#
+# Periodically check if the configuration has changed and reload the pipeline
+# This can also be triggered manually through the SIGHUP signal
+#
+# config.reload.automatic: false
+#
+# How often to check if the pipeline configuration has changed (in seconds)
+# Note that the unit value (s) is required. Values without a qualifier (e.g. 60)
+# are treated as nanoseconds.
+# Setting the interval this way is not recommended and might change in later versions.
+#
+# config.reload.interval: 3s
+#
+# Show fully compiled configuration as debug log message
+# NOTE: --log.level must be 'debug'
+#
+# config.debug: false
+#
+# When enabled, process escaped characters such as \n and \" in strings in the
+# pipeline configuration files.
+#
+# config.support_escapes: false
+#
+# ------------ API Settings -------------
+# Define settings related to the HTTP API here.
+#
+# The HTTP API is enabled by default. It can be disabled, but features that rely
+# on it will not work as intended.
+#
+# api.enabled: true
+#
+# By default, the HTTP API is not secured and is therefore bound to only the
+# host's loopback interface, ensuring that it is not accessible to the rest of
+# the network.
+# When secured with SSL and Basic Auth, the API is bound to _all_ interfaces
+# unless configured otherwise.
+#
+# api.http.host: 127.0.0.1
+#
+# The HTTP API web server will listen on an available port from the given range.
+# Values can be specified as a single port (e.g., `9600`), or an inclusive range
+# of ports (e.g., `9600-9700`).
+#
+# api.http.port: 9600-9700
+#
+# The HTTP API includes a customizable "environment" value in its response,
+# which can be configured here.
+#
+# api.environment: "production"
+#
+# The HTTP API can be secured with SSL (TLS). To do so, you will need to provide
+# the path to a password-protected keystore in p12 or jks format, along with credentials.
+#
+# api.ssl.enabled: false
+# api.ssl.keystore.path: /path/to/keystore.jks
+# api.ssl.keystore.password: "y0uRp4$$w0rD"
+#
+# The availability of SSL/TLS protocols depends on the JVM version. Certain protocols are
+# disabled by default and need to be enabled manually by changing `jdk.tls.disabledAlgorithms`
+# in the $JDK_HOME/conf/security/java.security configuration file.
+#
+# api.ssl.supported_protocols: [TLSv1.2,TLSv1.3]
+#
+# The HTTP API can be configured to require authentication. Acceptable values are
+#  - `none`:  no auth is required (default)
+#  - `basic`: clients must authenticate with HTTP Basic auth, as configured
+#             with `api.auth.basic.*` options below
+# api.auth.type: none
+#
+# When configured with `api.auth.type` `basic`, you must provide the credentials
+# that requests will be validated against. Usage of Environment or Keystore
+# variable replacements is encouraged (such as the value `"${HTTP_PASS}"`, which
+# resolves to the value stored in the keystore's `HTTP_PASS` variable if present
+# or the same variable from the environment)
+#
+# api.auth.basic.username: "logstash-user"
+# api.auth.basic.password: "s3cUreP4$$w0rD"
+#
+# When setting `api.auth.basic.password`, the password should meet
+# the default password policy requirements.
+# The default password policy requires non-empty minimum 8 char string that
+# includes a digit, upper case letter and lower case letter.
+# Policy mode sets Logstash to WARN or ERROR when HTTP authentication password doesn't
+# meet the password policy requirements.
+# The default is WARN. Setting to ERROR enforces stronger passwords (recommended).
+#
+# api.auth.basic.password_policy.mode: WARN
+#
+# ------------ Module Settings ---------------
+# Define modules here.  Modules definitions must be defined as an array.
+# The simple way to see this is to prepend each `name` with a `-`, and keep
+# all associated variables under the `name` they are associated with, and
+# above the next, like this:
+#
+# modules:
+#   - name: MODULE_NAME
+#     var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE
+#     var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE
+#     var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE
+#     var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE
+#
+# Module variable names must be in the format of
+#
+# var.PLUGIN_TYPE.PLUGIN_NAME.KEY
+#
+# modules:
+#
+# ------------ Cloud Settings ---------------
+# Define Elastic Cloud settings here.
+# Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy
+# and it may have an label prefix e.g. staging:dXMtZ...
+# This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host'
+# cloud.id: <identifier>
+#
+# Format of cloud.auth is: <user>:<pass>
+# This is optional
+# If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password'
+# If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password'
+# cloud.auth: elastic:<password>
+#
+# ------------ Queuing Settings --------------
+#
+# Internal queuing model, "memory" for legacy in-memory based queuing and
+# "persisted" for disk-based acked queueing. Defaults is memory
+#
+# queue.type: memory
+#
+# If `queue.type: persisted`, the directory path where the pipeline data files will be stored.
+# Each pipeline will group its PQ files in a subdirectory matching its `pipeline.id`.
+# Default is path.data/queue.
+#
+# path.queue:
+#
+# If using queue.type: persisted, the page data files size. The queue data consists of
+# append-only data files separated into pages. Default is 64mb
+#
+# queue.page_capacity: 64mb
+#
+# If using queue.type: persisted, the maximum number of unread events in the queue.
+# Default is 0 (unlimited)
+#
+# queue.max_events: 0
+#
+# If using queue.type: persisted, the total capacity of the queue in number of bytes.
+# If you would like more unacked events to be buffered in Logstash, you can increase the
+# capacity using this setting. Please make sure your disk drive has capacity greater than
+# the size specified here. If both max_bytes and max_events are specified, Logstash will pick
+# whichever criteria is reached first
+# Default is 1024mb or 1gb
+#
+# queue.max_bytes: 1024mb
+#
+# If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint
+# Default is 1024, 0 for unlimited
+#
+# queue.checkpoint.acks: 1024
+#
+# If using queue.type: persisted, the maximum number of written events before forcing a checkpoint
+# Default is 1024, 0 for unlimited
+#
+# queue.checkpoint.writes: 1024
+#
+# If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page
+# Default is 1000, 0 for no periodic checkpoint.
+#
+# queue.checkpoint.interval: 1000
+#
+# ------------ Dead-Letter Queue Settings --------------
+# Flag to turn on dead-letter queue.
+#
+# dead_letter_queue.enable: false
+
+# If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries
+# will be dropped if they would increase the size of the dead letter queue beyond this setting.
+# Default is 1024mb
+# dead_letter_queue.max_bytes: 1024mb
+
+# If using dead_letter_queue.enable: true, the interval in milliseconds where if no further events eligible for the DLQ
+# have been created, a dead letter queue file will be written. A low value here will mean that more, smaller, queue files
+# may be written, while a larger value will introduce more latency between items being "written" to the dead letter queue, and
+# being available to be read by the dead_letter_queue input when items are written infrequently.
+# Default is 5000.
+#
+# dead_letter_queue.flush_interval: 5000
+
+# If using dead_letter_queue.enable: true, controls which entries should be dropped to avoid exceeding the size limit.
+# Set the value to `drop_newer` (default) to stop accepting new events that would push the DLQ size over the limit.
+# Set the value to `drop_older` to remove queue pages containing the oldest events to make space for new ones.
+#
+# dead_letter_queue.storage_policy: drop_newer
+
+# If using dead_letter_queue.enable: true, the interval that events have to be considered valid. After the interval has
+# expired the events could be automatically deleted from the DLQ.
+# The interval could be expressed in days, hours, minutes or seconds, using as postfix notation like 5d,
+# to represent a five days interval.
+# The available units are respectively d, h, m, s for day, hours, minutes and seconds.
+# If not specified then the DLQ doesn't use any age policy for cleaning events.
+#
+# dead_letter_queue.retain.age: 1d
+
+# If using dead_letter_queue.enable: true, the directory path where the data files will be stored.
+# Default is path.data/dead_letter_queue
+#
+# path.dead_letter_queue:
+#
+# ------------ Debugging Settings --------------
+#
+# Options for log.level:
+#   * fatal
+#   * error
+#   * warn
+#   * info (default)
+#   * debug
+#   * trace
+# log.level: info
+#
+# Options for log.format:
+#   * plain (default)
+#   * json
+#
+# log.format: plain
+# log.format.json.fix_duplicate_message_fields: false
+#
+path.logs: /var/log/logstash
+#
+# ------------ Other Settings --------------
+#
+# Allow or block running Logstash as superuser (default: true)
+# allow_superuser: false
+#
+# Where to find custom plugins
+# path.plugins: []
+#
+# Flag to output log lines of each pipeline in its separate log file. Each log filename contains the pipeline.name
+# Default is false
+# pipeline.separate_logs: false
+#
+# Determine where to allocate memory buffers, for plugins that leverage them.
+# Default to direct, optionally can be switched to heap to select Java heap space.
+# pipeline.buffer.type: heap
+#
+# ------------ X-Pack Settings (not applicable for OSS build)--------------
+#
+# X-Pack Monitoring
+# https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html
+#xpack.monitoring.enabled: false
+#xpack.monitoring.elasticsearch.username: logstash_system
+#xpack.monitoring.elasticsearch.password: password
+#xpack.monitoring.elasticsearch.proxy: ["http://proxy:port"]
+#xpack.monitoring.elasticsearch.hosts: ["https://es1:9200", "https://es2:9200"]
+# an alternative to hosts + username/password settings is to use cloud_id/cloud_auth
+#xpack.monitoring.elasticsearch.cloud_id: monitoring_cluster_id:xxxxxxxxxx
+#xpack.monitoring.elasticsearch.cloud_auth: logstash_system:password
+# another authentication alternative is to use an Elasticsearch API key
+#xpack.monitoring.elasticsearch.api_key: "id:api_key"
+#xpack.monitoring.elasticsearch.ssl.certificate_authority: "/path/to/ca.crt"
+#xpack.monitoring.elasticsearch.ssl.ca_trusted_fingerprint: xxxxxxxxxx
+#xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file
+#xpack.monitoring.elasticsearch.ssl.truststore.password: password
+# use either keystore.path/keystore.password or certificate/key configurations
+#xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file
+#xpack.monitoring.elasticsearch.ssl.keystore.password: password
+#xpack.monitoring.elasticsearch.ssl.certificate: /path/to/file
+#xpack.monitoring.elasticsearch.ssl.key: /path/to/key
+#xpack.monitoring.elasticsearch.ssl.verification_mode: full
+#xpack.monitoring.elasticsearch.ssl.cipher_suites: []
+#xpack.monitoring.elasticsearch.sniffing: false
+#xpack.monitoring.collection.interval: 10s
+#xpack.monitoring.collection.pipeline.details.enabled: true
+#
+# X-Pack Management
+# https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html
+#xpack.management.enabled: false
+#xpack.management.pipeline.id: ["main", "apache_logs"]
+#xpack.management.elasticsearch.username: logstash_admin_user
+#xpack.management.elasticsearch.password: password
+#xpack.management.elasticsearch.proxy: ["http://proxy:port"]
+#xpack.management.elasticsearch.hosts: ["https://es1:9200", "https://es2:9200"]
+# an alternative to hosts + username/password settings is to use cloud_id/cloud_auth
+#xpack.management.elasticsearch.cloud_id: management_cluster_id:xxxxxxxxxx
+#xpack.management.elasticsearch.cloud_auth: logstash_admin_user:password
+# another authentication alternative is to use an Elasticsearch API key
+#xpack.management.elasticsearch.api_key: "id:api_key"
+#xpack.management.elasticsearch.ssl.ca_trusted_fingerprint: xxxxxxxxxx
+#xpack.management.elasticsearch.ssl.certificate_authority: "/path/to/ca.crt"
+#xpack.management.elasticsearch.ssl.truststore.path: /path/to/file
+#xpack.management.elasticsearch.ssl.truststore.password: password
+# use either keystore.path/keystore.password or certificate/key configurations
+#xpack.management.elasticsearch.ssl.keystore.path: /path/to/file
+#xpack.management.elasticsearch.ssl.keystore.password: password
+#xpack.management.elasticsearch.ssl.certificate: /path/to/file
+#xpack.management.elasticsearch.ssl.key: /path/to/certificate_key_file
+#xpack.management.elasticsearch.ssl.cipher_suites: []
+#xpack.management.elasticsearch.ssl.verification_mode: full
+#xpack.management.elasticsearch.sniffing: false
+#xpack.management.logstash.poll_interval: 5s
+
+# X-Pack GeoIP Database Management
+# https://www.elastic.co/guide/en/logstash/current/plugins-filters-geoip.html#plugins-filters-geoip-manage_update
+#xpack.geoip.downloader.enabled: true
+#xpack.geoip.downloader.endpoint: "https://geoip.elastic.co/v1/database"

logstash/pipelines.yml

@@ -0,0 +1,6 @@
+# This file is where you define your pipelines. You can define multiple.
+# For more information on multiple pipelines, see the documentation:
+#   https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html
+
+- pipeline.id: main
+  path.config: "/etc/logstash/conf.d/*.conf"

metricbeat/metricbeat.yml

@@ -0,0 +1,196 @@
+###################### Metricbeat Configuration Example #######################
+
+# This file is an example configuration file highlighting only the most common
+# options. The metricbeat.reference.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/metricbeat/index.html
+
+# =========================== Modules configuration ============================
+
+metricbeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  #reload.period: 10s
+
+# ======================= Elasticsearch template setting =======================
+
+setup.template.settings:
+  index.number_of_shards: 1
+  index.codec: best_compression
+  #_source.enabled: false
+
+setup.template.enabled: false
+setup.ilm.enabled: false
+
+# ================================== General ===================================
+
+# The name of the shipper that publishes the network data. It can be used to group
+# all the transactions sent by a single shipper in the web interface.
+#name:
+
+# The tags of the shipper are included in their field with each
+# transaction published.
+#tags: ["service-X", "web-tier"]
+
+# Optional fields that you can specify to add additional information to the
+# output.
+fields:
+  env: production
+  type: "sysmetrics"
+
+# ================================= Dashboards =================================
+# These settings control loading the sample dashboards to the Kibana index. Loading
+# the dashboards is disabled by default and can be enabled either by setting the
+# options here or by using the `setup` command.
+#setup.dashboards.enabled: false
+
+# The URL from where to download the dashboard archive. By default, this URL
+# has a value that is computed based on the Beat name and version. For released
+# versions, this URL points to the dashboard archive on the artifacts.elastic.co
+# website.
+#setup.dashboards.url:
+
+# =================================== Kibana ===================================
+
+# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
+# This requires a Kibana endpoint configuration.
+setup.kibana:
+
+  # Kibana Host
+  # Scheme and port can be left out and will be set to the default (http and 5601)
+  # In case you specify and additional path, the scheme is required: http://localhost:5601/path
+  # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
+  #host: "localhost:5601"
+
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
+# =============================== Elastic Cloud ================================
+
+# These settings simplify using Metricbeat with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
+# ================================== Outputs ===================================
+
+# Configure what output to use when sending the data collected by the beat.
+
+# ---------------------------- Elasticsearch Output ----------------------------
+output.elasticsearch:
+  # Array of hosts to connect to.
+  hosts: ["localhost:9200"]
+
+  # Performance preset - one of "balanced", "throughput", "scale",
+  # "latency", or "custom".
+  preset: balanced
+
+
+  # Protocol - either `http` (default) or `https`.
+  #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
+  username: "replace-user"
+  password: "replace-password"
+
+# ------------------------------ Logstash Output -------------------------------
+#output.logstash:
+  # The Logstash hosts
+  #hosts: ["localhost:5044"]
+
+  # Optional SSL. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+# ================================= Processors =================================
+
+# Configure processors to enhance or manipulate events generated by the beat.
+
+processors:
+  - add_host_metadata: ~
+  - add_cloud_metadata: ~
+  - add_docker_metadata: ~
+  - add_kubernetes_metadata: ~
+
+
+# ================================== Logging ===================================
+
+# Sets log level. The default log level is info.
+# Available log levels are: error, warning, info, debug
+#logging.level: debug
+
+# At debug level, you can selectively enable logging only for some components.
+# To enable all selectors, use ["*"]. Examples of other selectors are "beat",
+# "publisher", "service".
+#logging.selectors: ["*"]
+
+# ============================= X-Pack Monitoring ==============================
+# Metricbeat can export internal metrics to a central Elasticsearch monitoring
+# cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The
+# reporting is disabled by default.
+
+# Set to true to enable the monitoring reporter.
+#monitoring.enabled: false
+
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Metricbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
+# Uncomment to send the metrics to Elasticsearch. Most settings from the
+# Elasticsearch outputs are accepted here as well.
+# Note that the settings should point to your Elasticsearch *monitoring* cluster.
+# Any setting that is not set is automatically inherited from the Elasticsearch
+# output configuration, so if you have the Elasticsearch output configured such
+# that it is pointing to your Elasticsearch monitoring cluster, you can simply
+# uncomment the following line.
+#monitoring.elasticsearch:
+
+# ============================== Instrumentation ===============================
+
+# Instrumentation support for the metricbeat.
+#instrumentation:
+    # Set to true to enable instrumentation of metricbeat.
+    #enabled: false
+
+    # Environment in which metricbeat is running on (eg: staging, production, etc.)
+    #environment: ""
+
+    # APM Server hosts to report instrumentation results to.
+    #hosts:
+    #  - http://localhost:8200
+
+    # API Key for the APM Server(s).
+    # If api_key is set then secret_token will be ignored.
+    #api_key:
+
+    # Secret token for the APM Server(s).
+    #secret_token:
+
+
+# ================================= Migration ==================================
+
+# This allows to enable 6.7 migration aliases
+#migration.6_to_7.enabled: true

metricbeat/modules.d/system.yml

@@ -0,0 +1,45 @@
+# Module: system
+# Docs: https://www.elastic.co/guide/en/beats/metricbeat/8.18/metricbeat-module-system.html
+
+- module: system
+  period: 10s
+  metricsets:
+    - cpu
+    - load
+    - memory
+    - network
+    - process
+    - process_summary
+    - socket_summary
+    #- entropy
+    #- core
+    #- diskio
+    #- socket
+    #- service
+    #- users
+  process.include_top_n:
+    by_cpu: 5      # include top 5 processes by CPU
+    by_memory: 5   # include top 5 processes by memory
+  degrade_on_partial: false # mark metricset as degraded if partial metrics are emitted
+# Configure the mount point of the host’s filesystem for use in monitoring a host from within a container
+# hostfs: "/hostfs"
+
+- module: system
+  period: 1m
+  metricsets:
+    - filesystem
+    - fsstat
+  processors:
+  - drop_event.when.regexp:
+      system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)'
+
+- module: system
+  period: 15m
+  metricsets:
+    - uptime
+
+#- module: system
+#  period: 5m
+#  metricsets:
+#    - raid
+#  raid.mount_point: '/'

orchestrator/setup-crontab.sh

@@ -5,9 +5,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 # Configuration
 SCRIPTS=(
-    "../metrics/disk-usage.sh"
-    "../metrics/power-usage.sh" 
-    "../metrics/temperature.sh"
+    "../custom-metrics/disk-usage.sh"
+    "../custom-metrics/power-usage.sh" 
+    "../custom-metrics/temperature.sh"
 )
 
 CheckScriptExists() {
@@ -184,7 +184,7 @@ ShowUsage() {
     echo "  help     - Show this help message"
     echo ""
     echo "This script manages crontab entries for monitoring scripts:"
-    echo "$(printf '  %s\n' "${SCRIPTS[@]}" | sed 's|../metrics/||')"
+    echo "$(printf '  %s\n' "${SCRIPTS[@]}" | sed 's|../custom-metrics/||')"
     echo ""
     echo "Examples:"
     echo "  $0 install"

orchestrator/setup-filebeat.sh

@@ -0,0 +1,257 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. "${SCRIPT_DIR}/../share/logging.sh"
+
+# Configuration
+FILEBEAT_CONFIG_DIR="${SCRIPT_DIR}/../filebeat"
+FILEBEAT_INSTALL_DIR="/etc/filebeat"
+FILEBEAT_MODULES_D_DIR="${FILEBEAT_INSTALL_DIR}/modules.d"
+CONFIG_FILE="filebeat.yml"
+
+CheckRoot() {
+    if [[ $EUID -ne 0 ]]; then
+        LogError "This script must be run as root (use sudo)"
+        exit 1
+    fi
+}
+
+CheckFilebeatInstalled() {
+    if ! command -v filebeat &> /dev/null; then
+        LogWarning "filebeat command not found in PATH"
+        LogWarning "Make sure Filebeat is installed and configured"
+    fi
+    
+    if [[ ! -d "$FILEBEAT_INSTALL_DIR" ]]; then
+        LogInformation "Creating Filebeat config directory: $FILEBEAT_INSTALL_DIR"
+        mkdir -p "$FILEBEAT_INSTALL_DIR"
+        chmod 755 "$FILEBEAT_INSTALL_DIR"
+    fi
+    
+    if [[ ! -d "$FILEBEAT_MODULES_D_DIR" ]]; then
+        LogInformation "Creating Filebeat modules.d directory: $FILEBEAT_MODULES_D_DIR"
+        mkdir -p "$FILEBEAT_MODULES_D_DIR"
+        chmod 755 "$FILEBEAT_MODULES_D_DIR"
+    fi
+}
+
+UpdateFilebeatConfig() {
+    local user="$1"
+    local password="$2"
+    local config_file="${FILEBEAT_CONFIG_DIR}/${CONFIG_FILE}"
+    local temp_file=$(mktemp)
+    
+    if [[ ! -f "$config_file" ]]; then
+        LogError "Filebeat config file not found: $config_file"
+        exit 1
+    fi
+    
+    LogInformation "Updating Elasticsearch configuration in $CONFIG_FILE"
+    
+    sed -e "s/username: \"[^\"]*\"/username: \"$user\"/" \
+        -e "s/password: \"[^\"]*\"/password: \"$password\"/" \
+        "$config_file" > "$temp_file"
+    
+    if grep -q "username: \"$user\"" "$temp_file" && \
+       grep -q "password: \"$password\"" "$temp_file"; then
+        mv "$temp_file" "$config_file"
+        LogInformation "Updated Elasticsearch settings: user=$user"
+    else
+        rm -f "$temp_file"
+        LogError "Failed to update Elasticsearch configuration"
+        exit 1
+    fi
+}
+
+InstallFilebeatConfigs() {
+    local installed_count=0
+    local failed_count=0
+    
+    # Install main configuration file
+    LogInformation "Installing main Filebeat configuration file"
+    
+    # Install filebeat.yml
+    if [[ -f "${FILEBEAT_CONFIG_DIR}/${CONFIG_FILE}" ]]; then
+        LogInformation "Installing ${CONFIG_FILE}..."
+        if cp "${FILEBEAT_CONFIG_DIR}/${CONFIG_FILE}" "${FILEBEAT_INSTALL_DIR}/${CONFIG_FILE}" && chmod 644 "${FILEBEAT_INSTALL_DIR}/${CONFIG_FILE}"; then
+            LogInformation "Installed: ${FILEBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+            ((installed_count++))
+        else
+            LogError "Failed to install: ${CONFIG_FILE}"
+            ((failed_count++))
+        fi
+    else
+        LogWarning "${CONFIG_FILE} not found in source directory"
+    fi
+    
+    # Install modules.d configurations
+    LogInformation "Installing Filebeat modules from ${FILEBEAT_CONFIG_DIR}/modules"
+    if [[ -d "${FILEBEAT_CONFIG_DIR}/modules" ]]; then
+        while IFS= read -r -d '' module_file; do
+            local filename=$(basename "$module_file")
+            local target_file="${FILEBEAT_MODULES_D_DIR}/${filename}"
+            
+            LogInformation "Installing module $filename..."
+            if cp "$module_file" "$target_file" && chmod 644 "$target_file"; then
+                LogInformation "Installed: $target_file"
+                ((installed_count++))
+            else
+                LogError "Failed to install: $filename"
+                ((failed_count++))
+            fi
+        done < <(find "${FILEBEAT_CONFIG_DIR}/modules" -name "*.yml" -type f -print0)
+    else
+        LogWarning "modules directory not found in source"
+    fi
+    
+    LogInformation "Installation summary: $installed_count installed, $failed_count failed"
+    if [[ $failed_count -gt 0 ]]; then
+        exit 1
+    fi
+}
+
+ValidateFilebeatConfig() {
+    LogInformation "Validating Filebeat configuration..."
+    if command -v filebeat &> /dev/null; then
+        if filebeat test config -c "${FILEBEAT_INSTALL_DIR}/${CONFIG_FILE}" 2>/dev/null; then
+            LogInformation "Filebeat configuration validation passed"
+        else
+            LogWarning "Filebeat configuration validation failed or unavailable"
+            LogWarning "Run manually: sudo filebeat test config -c ${FILEBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+        fi
+    else
+        LogWarning "Filebeat not available for config validation"
+    fi
+}
+
+ShowStatus() {
+    LogInformation "Filebeat configuration status:"
+    LogInformation ""
+    
+    # Check main config file
+    if [[ -f "${FILEBEAT_INSTALL_DIR}/${CONFIG_FILE}" ]]; then
+        LogInformation "✓ Main config: ${FILEBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+    else
+        LogWarning "✗ Main config not found: ${FILEBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+    fi
+    
+    # Check modules.d directory
+    if [[ -d "$FILEBEAT_MODULES_D_DIR" ]]; then
+        local module_count=$(find "$FILEBEAT_MODULES_D_DIR" -name "*.yml" -type f | wc -l)
+        LogInformation "Module configurations: $module_count"
+        
+        if [[ $module_count -gt 0 ]]; then
+            LogInformation "Module files:"
+            find "$FILEBEAT_MODULES_D_DIR" -name "*.yml" -type f -exec basename {} \; | sort | while read -r file; do
+                LogInformation "  ✓ $file"
+            done
+        fi
+    else
+        LogWarning "Filebeat modules.d directory not found: $FILEBEAT_MODULES_D_DIR"
+    fi
+    
+    LogInformation ""
+    LogInformation "Commands:"
+    LogInformation "  Test config: sudo filebeat test config -c ${FILEBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+    LogInformation "  Test output: sudo filebeat test output -c ${FILEBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+    LogInformation "  Start Filebeat: sudo systemctl start filebeat"
+    LogInformation "  Check status: sudo systemctl status filebeat"
+}
+
+RemoveFilebeatConfigs() {
+    local removed_count=0
+    
+    LogInformation "Removing Filebeat configurations..."
+    
+    # Remove main config file
+    if [[ -f "${FILEBEAT_INSTALL_DIR}/${CONFIG_FILE}" ]]; then
+        LogInformation "Removing ${CONFIG_FILE}..."
+        rm -f "${FILEBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+        ((removed_count++))
+    fi
+    
+    # Remove modules.d configurations
+    if [[ -d "$FILEBEAT_MODULES_D_DIR" ]]; then
+        while IFS= read -r -d '' source_file; do
+            local filename=$(basename "$source_file")
+            local target_file="${FILEBEAT_MODULES_D_DIR}/${filename}"
+            
+            if [[ -f "$target_file" ]]; then
+                LogInformation "Removing module $filename..."
+                rm -f "$target_file"
+                ((removed_count++))
+            fi
+        done < <(find "${FILEBEAT_CONFIG_DIR}/modules" -name "*.yml" -type f -print0 2>/dev/null)
+    fi
+    
+    if [[ $removed_count -gt 0 ]]; then
+        LogInformation "Removed $removed_count Filebeat configurations"
+    else
+        LogWarning "No configurations were found to remove"
+    fi
+}
+
+ShowUsage() {
+    echo "Usage: $0 install <user> <password>"
+    echo "       $0 [remove|status|help]"
+    echo ""
+    echo "Commands:"
+    echo "  install  - Install Filebeat configurations with Elasticsearch settings"
+    echo "  remove   - Remove all Filebeat configurations"
+    echo "  status   - Show status of configurations"
+    echo "  help     - Show this help message"
+    echo ""
+    echo "Parameters for install:"
+    echo "  user     - Elasticsearch username"
+    echo "  password - Elasticsearch password"
+    echo ""
+    echo "Examples:"
+    echo "  sudo $0 install elastic mypassword"
+    echo "  sudo $0 status"
+    echo "  sudo $0 remove"
+}
+
+Main() {
+    local action="$1"
+    
+    case "$action" in
+        install)
+            if [[ $# -ne 3 ]]; then
+                LogError "Install command requires 2 parameters: user, password"
+                ShowUsage
+                exit 1
+            fi
+            
+            local user="$2"
+            local password="$3"
+            
+            LogInformation "Installing Filebeat configurations"
+            LogInformation "Elasticsearch settings: user=$user"
+            
+            CheckRoot
+            CheckFilebeatInstalled
+            UpdateFilebeatConfig "$user" "$password"
+            InstallFilebeatConfigs
+            ValidateFilebeatConfig
+            LogInformation "Filebeat setup completed successfully!"
+            ;;
+        remove)
+            LogInformation "Removing Filebeat configurations"
+            CheckRoot
+            RemoveFilebeatConfigs
+            ;;
+        status)
+            ShowStatus
+            ;;
+        help|--help|-h)
+            ShowUsage
+            ;;
+        *)
+            LogError "Unknown action: $action"
+            ShowUsage
+            exit 1
+            ;;
+    esac
+}
+
+Main "$@"

orchestrator/setup-logstash.sh

@@ -5,8 +5,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 # Configuration
 LOGSTASH_CONFIG_DIR="${SCRIPT_DIR}/../logstash"
-LOGSTASH_INSTALL_DIR="/etc/logstash/conf.d"
-OUTPUT_CONFIG_FILE="90-output.conf"
+LOGSTASH_INSTALL_DIR="/etc/logstash"
+LOGSTASH_CONF_D_DIR="${LOGSTASH_INSTALL_DIR}/conf.d"
+OUTPUT_CONFIG_FILE="conf.d/90-output.conf"
 
 CheckRoot() {
     if [[ $EUID -ne 0 ]]; then
@@ -26,12 +27,17 @@ CheckLogstashInstalled() {
         mkdir -p "$LOGSTASH_INSTALL_DIR"
         chmod 755 "$LOGSTASH_INSTALL_DIR"
     fi
+    
+    if [[ ! -d "$LOGSTASH_CONF_D_DIR" ]]; then
+        LogInformation "Creating Logstash conf.d directory: $LOGSTASH_CONF_D_DIR"
+        mkdir -p "$LOGSTASH_CONF_D_DIR"
+        chmod 755 "$LOGSTASH_CONF_D_DIR"
+    fi
 }
 
 UpdateOutputConfig() {
-    local index="$1"
-    local user="$2"
-    local password="$3"
+    local user="$1"
+    local password="$2"
     local output_file="${LOGSTASH_CONFIG_DIR}/${OUTPUT_CONFIG_FILE}"
     local temp_file=$(mktemp)
     
@@ -42,16 +48,14 @@ UpdateOutputConfig() {
     
     LogInformation "Updating Elasticsearch configuration in $OUTPUT_CONFIG_FILE"
     
-    sed -e "s/index => \"[^\"]*\"/index => \"$index\"/" \
-        -e "s/user => \"[^\"]*\"/user => \"$user\"/" \
+    sed -e "s/user => \"[^\"]*\"/user => \"$user\"/" \
         -e "s/password => \"[^\"]*\"/password => \"$password\"/" \
         "$output_file" > "$temp_file"
     
-    if grep -q "index => \"$index\"" "$temp_file" && \
-       grep -q "user => \"$user\"" "$temp_file" && \
+    if grep -q "user => \"$user\"" "$temp_file" && \
        grep -q "password => \"$password\"" "$temp_file"; then
         mv "$temp_file" "$output_file"
-        LogInformation "Updated Elasticsearch settings: index=$index, user=$user"
+        LogInformation "Updated Elasticsearch settings: user=$user"
     else
         rm -f "$temp_file"
         LogError "Failed to update Elasticsearch configuration"
@@ -63,20 +67,56 @@ InstallLogstashConfigs() {
     local installed_count=0
     local failed_count=0
     
-    LogInformation "Installing Logstash configurations from $LOGSTASH_CONFIG_DIR"
-    while IFS= read -r -d '' config_file; do
-        local filename=$(basename "$config_file")
-        local target_file="${LOGSTASH_INSTALL_DIR}/${filename}"
-        
-        LogInformation "Installing $filename..."
-        if cp "$config_file" "$target_file" && chmod 644 "$target_file"; then
-            LogInformation "Installed: $target_file"
+    # Install main configuration files
+    LogInformation "Installing main Logstash configuration files"
+    
+    # Install logstash.yml
+    if [[ -f "${LOGSTASH_CONFIG_DIR}/logstash.yml" ]]; then
+        LogInformation "Installing logstash.yml..."
+        if cp "${LOGSTASH_CONFIG_DIR}/logstash.yml" "${LOGSTASH_INSTALL_DIR}/logstash.yml" && chmod 644 "${LOGSTASH_INSTALL_DIR}/logstash.yml"; then
+            LogInformation "Installed: ${LOGSTASH_INSTALL_DIR}/logstash.yml"
             ((installed_count++))
         else
-            LogError "Failed to install: $filename"
+            LogError "Failed to install: logstash.yml"
             ((failed_count++))
         fi
-    done < <(find "$LOGSTASH_CONFIG_DIR" -name "*.conf" -type f -print0)
+    else
+        LogWarning "logstash.yml not found in source directory"
+    fi
+    
+    # Install pipelines.yml
+    if [[ -f "${LOGSTASH_CONFIG_DIR}/pipelines.yml" ]]; then
+        LogInformation "Installing pipelines.yml..."
+        if cp "${LOGSTASH_CONFIG_DIR}/pipelines.yml" "${LOGSTASH_INSTALL_DIR}/pipelines.yml" && chmod 644 "${LOGSTASH_INSTALL_DIR}/pipelines.yml"; then
+            LogInformation "Installed: ${LOGSTASH_INSTALL_DIR}/pipelines.yml"
+            ((installed_count++))
+        else
+            LogError "Failed to install: pipelines.yml"
+            ((failed_count++))
+        fi
+    else
+        LogWarning "pipelines.yml not found in source directory"
+    fi
+    
+    # Install conf.d configurations
+    LogInformation "Installing Logstash pipeline configurations from ${LOGSTASH_CONFIG_DIR}/conf.d"
+    if [[ -d "${LOGSTASH_CONFIG_DIR}/conf.d" ]]; then
+        while IFS= read -r -d '' config_file; do
+            local filename=$(basename "$config_file")
+            local target_file="${LOGSTASH_CONF_D_DIR}/${filename}"
+            
+            LogInformation "Installing $filename..."
+            if cp "$config_file" "$target_file" && chmod 644 "$target_file"; then
+                LogInformation "Installed: $target_file"
+                ((installed_count++))
+            else
+                LogError "Failed to install: $filename"
+                ((failed_count++))
+            fi
+        done < <(find "${LOGSTASH_CONFIG_DIR}/conf.d" -name "*.conf" -type f -print0)
+    else
+        LogWarning "conf.d directory not found in source"
+    fi
     
     LogInformation "Installation summary: $installed_count installed, $failed_count failed"
     if [[ $failed_count -gt 0 ]]; then
@@ -102,16 +142,32 @@ ShowStatus() {
     LogInformation "Logstash configuration status:"
     LogInformation ""
     
-    if [[ -d "$LOGSTASH_INSTALL_DIR" ]]; then
-        local config_count=$(find "$LOGSTASH_INSTALL_DIR" -name "*.conf" -type f | wc -l)
-        LogInformation "Installed configurations: $config_count"
-        
-        LogInformation "Configuration files:"
-        find "$LOGSTASH_INSTALL_DIR" -name "*.conf" -type f -exec basename {} \; | sort | while read -r file; do
-            LogInformation "  ✓ $file"
-        done
+    # Check main config files
+    if [[ -f "${LOGSTASH_INSTALL_DIR}/logstash.yml" ]]; then
+        LogInformation "✓ Main config: ${LOGSTASH_INSTALL_DIR}/logstash.yml"
     else
-        LogWarning "Logstash config directory not found: $LOGSTASH_INSTALL_DIR"
+        LogWarning "✗ Main config not found: ${LOGSTASH_INSTALL_DIR}/logstash.yml"
+    fi
+    
+    if [[ -f "${LOGSTASH_INSTALL_DIR}/pipelines.yml" ]]; then
+        LogInformation "✓ Pipelines config: ${LOGSTASH_INSTALL_DIR}/pipelines.yml"
+    else
+        LogWarning "✗ Pipelines config not found: ${LOGSTASH_INSTALL_DIR}/pipelines.yml"
+    fi
+    
+    # Check conf.d directory
+    if [[ -d "$LOGSTASH_CONF_D_DIR" ]]; then
+        local config_count=$(find "$LOGSTASH_CONF_D_DIR" -name "*.conf" -type f | wc -l)
+        LogInformation "Pipeline configurations: $config_count"
+        
+        if [[ $config_count -gt 0 ]]; then
+            LogInformation "Configuration files:"
+            find "$LOGSTASH_CONF_D_DIR" -name "*.conf" -type f -exec basename {} \; | sort | while read -r file; do
+                LogInformation "  ✓ $file"
+            done
+        fi
+    else
+        LogWarning "Logstash conf.d directory not found: $LOGSTASH_CONF_D_DIR"
     fi
     
     LogInformation ""
@@ -124,22 +180,34 @@ ShowStatus() {
 RemoveLogstashConfigs() {
     local removed_count=0
     
-    if [[ ! -d "$LOGSTASH_INSTALL_DIR" ]]; then
-        LogWarning "Logstash config directory not found: $LOGSTASH_INSTALL_DIR"
-        return
+    LogInformation "Removing Logstash configurations..."
+    
+    # Remove main config files
+    if [[ -f "${LOGSTASH_INSTALL_DIR}/logstash.yml" ]]; then
+        LogInformation "Removing logstash.yml..."
+        rm -f "${LOGSTASH_INSTALL_DIR}/logstash.yml"
+        ((removed_count++))
     fi
     
-    LogInformation "Removing Logstash configurations..."
-    while IFS= read -r -d '' source_file; do
-        local filename=$(basename "$source_file")
-        local target_file="${LOGSTASH_INSTALL_DIR}/${filename}"
-        
-        if [[ -f "$target_file" ]]; then
-            LogInformation "Removing $filename..."
-            rm -f "$target_file"
-            ((removed_count++))
-        fi
-    done < <(find "$LOGSTASH_CONFIG_DIR" -name "*.conf" -type f -print0)
+    if [[ -f "${LOGSTASH_INSTALL_DIR}/pipelines.yml" ]]; then
+        LogInformation "Removing pipelines.yml..."
+        rm -f "${LOGSTASH_INSTALL_DIR}/pipelines.yml"
+        ((removed_count++))
+    fi
+    
+    # Remove conf.d configurations
+    if [[ -d "$LOGSTASH_CONF_D_DIR" ]]; then
+        while IFS= read -r -d '' source_file; do
+            local filename=$(basename "$source_file")
+            local target_file="${LOGSTASH_CONF_D_DIR}/${filename}"
+            
+            if [[ -f "$target_file" ]]; then
+                LogInformation "Removing $filename..."
+                rm -f "$target_file"
+                ((removed_count++))
+            fi
+        done < <(find "${LOGSTASH_CONFIG_DIR}/conf.d" -name "*.conf" -type f -print0 2>/dev/null)
+    fi
     
     if [[ $removed_count -gt 0 ]]; then
         LogInformation "Removed $removed_count Logstash configurations"
@@ -149,7 +217,7 @@ RemoveLogstashConfigs() {
 }
 
 ShowUsage() {
-    echo "Usage: $0 install <index> <user> <password>"
+    echo "Usage: $0 install <user> <password>"
     echo "       $0 [remove|status|help]"
     echo ""
     echo "Commands:"
@@ -159,12 +227,11 @@ ShowUsage() {
     echo "  help     - Show this help message"
     echo ""
     echo "Parameters for install:"
-    echo "  index    - Elasticsearch index name"
     echo "  user     - Elasticsearch username"
     echo "  password - Elasticsearch password"
     echo ""
     echo "Examples:"
-    echo "  sudo $0 install myserver-logs elastic mypassword"
+    echo "  sudo $0 install elastic mypassword"
     echo "  sudo $0 status"
     echo "  sudo $0 remove"
 }
@@ -174,22 +241,21 @@ Main() {
     
     case "$action" in
         install)
-            if [[ $# -ne 4 ]]; then
-                LogError "Install command requires 3 parameters: index, user, password"
+            if [[ $# -ne 3 ]]; then
+                LogError "Install command requires 2 parameters: user, password"
                 ShowUsage
                 exit 1
             fi
             
-            local index="$2"
-            local user="$3"
-            local password="$4"
+            local user="$2"
+            local password="$3"
             
             LogInformation "Installing Logstash configurations"
-            LogInformation "Elasticsearch settings: index=$index, user=$user"
+            LogInformation "Elasticsearch settings: user=$user"
             
             CheckRoot
             CheckLogstashInstalled
-            UpdateOutputConfig "$index" "$user" "$password"
+            UpdateOutputConfig "$user" "$password"
             InstallLogstashConfigs
             ValidateLogstashConfig
             LogInformation "Logstash setup completed successfully!"

orchestrator/setup-metricbeat.sh

@@ -0,0 +1,257 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. "${SCRIPT_DIR}/../share/logging.sh"
+
+# Configuration
+METRICBEAT_CONFIG_DIR="${SCRIPT_DIR}/../metricbeat"
+METRICBEAT_INSTALL_DIR="/etc/metricbeat"
+METRICBEAT_MODULES_D_DIR="${METRICBEAT_INSTALL_DIR}/modules.d"
+CONFIG_FILE="metricbeat.yml"
+
+CheckRoot() {
+    if [[ $EUID -ne 0 ]]; then
+        LogError "This script must be run as root (use sudo)"
+        exit 1
+    fi
+}
+
+CheckMetricbeatInstalled() {
+    if ! command -v metricbeat &> /dev/null; then
+        LogWarning "metricbeat command not found in PATH"
+        LogWarning "Make sure Metricbeat is installed and configured"
+    fi
+    
+    if [[ ! -d "$METRICBEAT_INSTALL_DIR" ]]; then
+        LogInformation "Creating Metricbeat config directory: $METRICBEAT_INSTALL_DIR"
+        mkdir -p "$METRICBEAT_INSTALL_DIR"
+        chmod 755 "$METRICBEAT_INSTALL_DIR"
+    fi
+    
+    if [[ ! -d "$METRICBEAT_MODULES_D_DIR" ]]; then
+        LogInformation "Creating Metricbeat modules.d directory: $METRICBEAT_MODULES_D_DIR"
+        mkdir -p "$METRICBEAT_MODULES_D_DIR"
+        chmod 755 "$METRICBEAT_MODULES_D_DIR"
+    fi
+}
+
+UpdateMetricbeatConfig() {
+    local user="$1"
+    local password="$2"
+    local config_file="${METRICBEAT_CONFIG_DIR}/${CONFIG_FILE}"
+    local temp_file=$(mktemp)
+    
+    if [[ ! -f "$config_file" ]]; then
+        LogError "Metricbeat config file not found: $config_file"
+        exit 1
+    fi
+    
+    LogInformation "Updating Elasticsearch configuration in $CONFIG_FILE"
+    
+    sed -e "s/username: \"[^\"]*\"/username: \"$user\"/" \
+        -e "s/password: \"[^\"]*\"/password: \"$password\"/" \
+        "$config_file" > "$temp_file"
+    
+    if grep -q "username: \"$user\"" "$temp_file" && \
+       grep -q "password: \"$password\"" "$temp_file"; then
+        mv "$temp_file" "$config_file"
+        LogInformation "Updated Elasticsearch settings: user=$user"
+    else
+        rm -f "$temp_file"
+        LogError "Failed to update Elasticsearch configuration"
+        exit 1
+    fi
+}
+
+InstallMetricbeatConfigs() {
+    local installed_count=0
+    local failed_count=0
+    
+    # Install main configuration file
+    LogInformation "Installing main Metricbeat configuration file"
+    
+    # Install metricbeat.yml
+    if [[ -f "${METRICBEAT_CONFIG_DIR}/${CONFIG_FILE}" ]]; then
+        LogInformation "Installing ${CONFIG_FILE}..."
+        if cp "${METRICBEAT_CONFIG_DIR}/${CONFIG_FILE}" "${METRICBEAT_INSTALL_DIR}/${CONFIG_FILE}" && chmod 644 "${METRICBEAT_INSTALL_DIR}/${CONFIG_FILE}"; then
+            LogInformation "Installed: ${METRICBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+            ((installed_count++))
+        else
+            LogError "Failed to install: ${CONFIG_FILE}"
+            ((failed_count++))
+        fi
+    else
+        LogWarning "${CONFIG_FILE} not found in source directory"
+    fi
+    
+    # Install modules.d configurations
+    LogInformation "Installing Metricbeat modules from ${METRICBEAT_CONFIG_DIR}/modules.d"
+    if [[ -d "${METRICBEAT_CONFIG_DIR}/modules.d" ]]; then
+        while IFS= read -r -d '' module_file; do
+            local filename=$(basename "$module_file")
+            local target_file="${METRICBEAT_MODULES_D_DIR}/${filename}"
+            
+            LogInformation "Installing module $filename..."
+            if cp "$module_file" "$target_file" && chmod 644 "$target_file"; then
+                LogInformation "Installed: $target_file"
+                ((installed_count++))
+            else
+                LogError "Failed to install: $filename"
+                ((failed_count++))
+            fi
+        done < <(find "${METRICBEAT_CONFIG_DIR}/modules.d" -name "*.yml" -type f -print0)
+    else
+        LogWarning "modules.d directory not found in source"
+    fi
+    
+    LogInformation "Installation summary: $installed_count installed, $failed_count failed"
+    if [[ $failed_count -gt 0 ]]; then
+        exit 1
+    fi
+}
+
+ValidateMetricbeatConfig() {
+    LogInformation "Validating Metricbeat configuration..."
+    if command -v metricbeat &> /dev/null; then
+        if metricbeat test config -c "${METRICBEAT_INSTALL_DIR}/${CONFIG_FILE}" 2>/dev/null; then
+            LogInformation "Metricbeat configuration validation passed"
+        else
+            LogWarning "Metricbeat configuration validation failed or unavailable"
+            LogWarning "Run manually: sudo metricbeat test config -c ${METRICBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+        fi
+    else
+        LogWarning "Metricbeat not available for config validation"
+    fi
+}
+
+ShowStatus() {
+    LogInformation "Metricbeat configuration status:"
+    LogInformation ""
+    
+    # Check main config file
+    if [[ -f "${METRICBEAT_INSTALL_DIR}/${CONFIG_FILE}" ]]; then
+        LogInformation "✓ Main config: ${METRICBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+    else
+        LogWarning "✗ Main config not found: ${METRICBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+    fi
+    
+    # Check modules.d directory
+    if [[ -d "$METRICBEAT_MODULES_D_DIR" ]]; then
+        local module_count=$(find "$METRICBEAT_MODULES_D_DIR" -name "*.yml" -type f | wc -l)
+        LogInformation "Module configurations: $module_count"
+        
+        if [[ $module_count -gt 0 ]]; then
+            LogInformation "Module files:"
+            find "$METRICBEAT_MODULES_D_DIR" -name "*.yml" -type f -exec basename {} \; | sort | while read -r file; do
+                LogInformation "  ✓ $file"
+            done
+        fi
+    else
+        LogWarning "Metricbeat modules.d directory not found: $METRICBEAT_MODULES_D_DIR"
+    fi
+    
+    LogInformation ""
+    LogInformation "Commands:"
+    LogInformation "  Test config: sudo metricbeat test config -c ${METRICBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+    LogInformation "  Test output: sudo metricbeat test output -c ${METRICBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+    LogInformation "  Start Metricbeat: sudo systemctl start metricbeat"
+    LogInformation "  Check status: sudo systemctl status metricbeat"
+}
+
+RemoveMetricbeatConfigs() {
+    local removed_count=0
+    
+    LogInformation "Removing Metricbeat configurations..."
+    
+    # Remove main config file
+    if [[ -f "${METRICBEAT_INSTALL_DIR}/${CONFIG_FILE}" ]]; then
+        LogInformation "Removing ${CONFIG_FILE}..."
+        rm -f "${METRICBEAT_INSTALL_DIR}/${CONFIG_FILE}"
+        ((removed_count++))
+    fi
+    
+    # Remove modules.d configurations
+    if [[ -d "$METRICBEAT_MODULES_D_DIR" ]]; then
+        while IFS= read -r -d '' source_file; do
+            local filename=$(basename "$source_file")
+            local target_file="${METRICBEAT_MODULES_D_DIR}/${filename}"
+            
+            if [[ -f "$target_file" ]]; then
+                LogInformation "Removing module $filename..."
+                rm -f "$target_file"
+                ((removed_count++))
+            fi
+        done < <(find "${METRICBEAT_CONFIG_DIR}/modules.d" -name "*.yml" -type f -print0 2>/dev/null)
+    fi
+    
+    if [[ $removed_count -gt 0 ]]; then
+        LogInformation "Removed $removed_count Metricbeat configurations"
+    else
+        LogWarning "No configurations were found to remove"
+    fi
+}
+
+ShowUsage() {
+    echo "Usage: $0 install <user> <password>"
+    echo "       $0 [remove|status|help]"
+    echo ""
+    echo "Commands:"
+    echo "  install  - Install Metricbeat configurations with Elasticsearch settings"
+    echo "  remove   - Remove all Metricbeat configurations"
+    echo "  status   - Show status of configurations"
+    echo "  help     - Show this help message"
+    echo ""
+    echo "Parameters for install:"
+    echo "  user     - Elasticsearch username"
+    echo "  password - Elasticsearch password"
+    echo ""
+    echo "Examples:"
+    echo "  sudo $0 install elastic mypassword"
+    echo "  sudo $0 status"
+    echo "  sudo $0 remove"
+}
+
+Main() {
+    local action="$1"
+    
+    case "$action" in
+        install)
+            if [[ $# -ne 3 ]]; then
+                LogError "Install command requires 2 parameters: user, password"
+                ShowUsage
+                exit 1
+            fi
+            
+            local user="$2"
+            local password="$3"
+            
+            LogInformation "Installing Metricbeat configurations"
+            LogInformation "Elasticsearch settings: user=$user"
+            
+            CheckRoot
+            CheckMetricbeatInstalled
+            UpdateMetricbeatConfig "$user" "$password"
+            InstallMetricbeatConfigs
+            ValidateMetricbeatConfig
+            LogInformation "Metricbeat setup completed successfully!"
+            ;;
+        remove)
+            LogInformation "Removing Metricbeat configurations"
+            CheckRoot
+            RemoveMetricbeatConfigs
+            ;;
+        status)
+            ShowStatus
+            ;;
+        help|--help|-h)
+            ShowUsage
+            ;;
+        *)
+            LogError "Unknown action: $action"
+            ShowUsage
+            exit 1
+            ;;
+    esac
+}
+
+Main "$@"