diff --git a/logstash/10-input.conf b/logstash/10-input.conf new file mode 100644 index 0000000..3fc28dd --- /dev/null +++ b/logstash/10-input.conf @@ -0,0 +1,31 @@ +input { + file { + path => "/var/log/temperature.log" + start_position => "beginning" + sincedb_path => "/dev/null" + add_field => { + "type" => "temperature" + "host.hostname" => "praguetopbox" + } + } + + file { + path => "/var/log/power_usage.log" + start_position => "beginning" + sincedb_path => "/dev/null" + add_field => { + "type" => "power" + "host.hostname" => "praguetopbox" + } + } + + file { + path => "/var/log/disk_usage.log" + start_position => "beginning" + sincedb_path => "/dev/null" + add_field => { + "type" => "disk" + "host.hostname" => "praguetopbox" + } + } +} \ No newline at end of file diff --git a/logstash/20-temperature.conf b/logstash/20-temperature.conf new file mode 100644 index 0000000..0421d4d --- /dev/null +++ b/logstash/20-temperature.conf @@ -0,0 +1,18 @@ +filter { + if [type] == "temperature" { + dissect { + mapping => { + "message" => "%{timestamp} %{system.temperature.sensor_name} %{system.temperature.value}°C" + } + } + + date { + match => ["timestamp", "ISO8601"] + target => "@timestamp" + } + + mutate { + convert => { "system.temperature.value" => "float" } + } + } +} diff --git a/logstash/21-power-usage.conf b/logstash/21-power-usage.conf new file mode 100644 index 0000000..9df9bf1 --- /dev/null +++ b/logstash/21-power-usage.conf @@ -0,0 +1,18 @@ +filter { + if [type] == "power" { + dissect { + mapping => { + "message" => "%{timestamp} %{system.power.value}" + } + } + + date { + match => ["timestamp", "ISO8601"] + target => "@timestamp" + } + + mutate { + convert => { "system.power.value" => "float" } + } + } +} \ No newline at end of file diff --git a/logstash/22-disk-usage.conf b/logstash/22-disk-usage.conf new file mode 100644 index 0000000..d6b2365 --- /dev/null +++ b/logstash/22-disk-usage.conf @@ -0,0 +1,66 @@ +filter { + if [type] == "disk" { + dissect { + mapping => { + "message" => "%{timestamp} %{system.disk.mount_point} %{system.disk.used} %{system.disk.total}" + } + } + + date { + match => ["timestamp", "ISO8601"] + target => "@timestamp" + } + + mutate { + convert => { + "system.disk.used" => "integer" + "system.disk.total" => "integer" + } + } + + ruby { + code => " + used = event.get('system.disk.used').to_f + total = event.get('system.disk.total').to_f + if total > 0 + percentage = (used / total) * 100 + event.set('system.disk.usage_percent', percentage.round(2)) + end + " + } + + ruby { + code => " + def bytes_to_human(bytes) + units = ['B', 'KB', 'MB', 'GB', 'TB'] + return '0 B' if bytes == 0 + + exp = (Math.log(bytes) / Math.log(1024)).floor + exp = [exp, units.length - 1].min + + size = bytes / (1024.0 ** exp) + unit = units[exp] + + if size >= 100 + '%.0f %s' % [size, unit] + elsif size >= 10 + '%.1f %s' % [size, unit] + else + '%.2f %s' % [size, unit] + end + end + + used_bytes = event.get('system.disk.used') + total_bytes = event.get('system.disk.total') + + if used_bytes + event.set('system.disk.used_human', bytes_to_human(used_bytes)) + end + + if total_bytes + event.set('system.disk.total_human', bytes_to_human(total_bytes)) + end + " + } + } +} diff --git a/logstash/90-output.conf b/logstash/90-output.conf new file mode 100644 index 0000000..07a89d0 --- /dev/null +++ b/logstash/90-output.conf @@ -0,0 +1,12 @@ +output { + # Forward logs to Elasticsearch + elasticsearch { + hosts => ["http://localhost:9200"] + index => "replace-index" # Index pattern based on log type and date + user => "replace-user" + password => "replace-password" + } + if "_grokparsefailure" in [tags] { + stdout { codec => rubydebug } + } +} \ No newline at end of file diff --git a/orchestrator/setup-crontab.sh b/orchestrator/setup-crontab.sh index adef05e..c9bbbc8 100755 --- a/orchestrator/setup-crontab.sh +++ b/orchestrator/setup-crontab.sh @@ -3,6 +3,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "${SCRIPT_DIR}/../share/logging.sh" +# Configuration SCRIPTS=( "../metrics/disk-usage.sh" "../metrics/power-usage.sh" @@ -24,6 +25,35 @@ CheckScriptExists() { fi } +InstallCrontabEntries() { + local installed_count=0 + local failed_count=0 + + BackupCrontab + + for script in "${SCRIPTS[@]}"; do + LogInformation "Processing $script..." + + if CheckScriptExists "$script"; then + LogInformation "Script $script found and is executable" + if ManageCrontabEntry "$script"; then + ((installed_count++)) + else + ((failed_count++)) + fi + else + LogWarning "Script $script not found or not accessible, skipping..." + ((failed_count++)) + fi + done + + LogInformation "Installation summary: $installed_count installed, $failed_count failed" + + if [[ $failed_count -gt 0 ]]; then + exit 1 + fi +} + ManageCrontabEntry() { local script="$1" local script_path="${SCRIPT_DIR}/${script}" @@ -37,19 +67,100 @@ ManageCrontabEntry() { if echo "$current_crontab" | grep -q "^\* \* \* \* \* ${script_path}"; then LogInformation "Entry for $script is already set to run every minute" + return 0 else LogInformation "Updating $script to run every minute" local new_crontab new_crontab=$(echo "$current_crontab" | grep -v "$script_path") new_crontab="${new_crontab}${new_crontab:+$'\n'}${cron_pattern}" - echo "$new_crontab" | crontab - - LogInformation "Updated crontab entry for $script" + if echo "$new_crontab" | crontab -; then + LogInformation "Updated crontab entry for $script" + return 0 + else + LogError "Failed to update crontab entry for $script" + return 1 + fi fi else LogInformation "Adding new crontab entry for $script" local new_crontab="${current_crontab}${current_crontab:+$'\n'}${cron_pattern}" - echo "$new_crontab" | crontab - - LogInformation "Added crontab entry for $script" + if echo "$new_crontab" | crontab -; then + LogInformation "Added crontab entry for $script" + return 0 + else + LogError "Failed to add crontab entry for $script" + return 1 + fi + fi +} + +ShowStatus() { + LogInformation "Crontab entries status:" + LogInformation "" + + local found_entries=0 + for script in "${SCRIPTS[@]}"; do + local script_path="${SCRIPT_DIR}/${script}" + local script_name=$(basename "$script") + + if crontab -l 2>/dev/null | grep -q "$script_path"; then + LogInformation "✓ $script_name - INSTALLED" + ((found_entries++)) + elif [[ -f "$script_path" ]]; then + LogInformation "✗ $script_name - NOT INSTALLED (script available)" + else + LogInformation "✗ $script_name - SCRIPT MISSING" + fi + done + + LogInformation "" + if [[ $found_entries -gt 0 ]]; then + LogInformation "Commands:" + LogInformation " View crontab: crontab -l" + LogInformation " Edit crontab: crontab -e" + LogInformation " Check logs: sudo tail -f /var/log/syslog | grep CRON" + else + LogWarning "No crontab entries are currently installed" + fi +} + +RemoveCrontabEntries() { + local removed_count=0 + + local current_crontab + current_crontab=$(crontab -l 2>/dev/null || echo "") + + if [[ -z "$current_crontab" ]]; then + LogWarning "No crontab entries found to remove" + return + fi + + local new_crontab="$current_crontab" + + for script in "${SCRIPTS[@]}"; do + local script_path="${SCRIPT_DIR}/${script}" + local script_name=$(basename "$script") + + if echo "$new_crontab" | grep -q "$script_path"; then + LogInformation "Removing crontab entry for $script_name..." + new_crontab=$(echo "$new_crontab" | grep -v "$script_path") + ((removed_count++)) + else + LogDebug "No crontab entry found for $script_name" + fi + done + + if [[ $removed_count -gt 0 ]]; then + if [[ -z "$new_crontab" ]]; then + crontab -r 2>/dev/null || true + LogInformation "Removed all entries, crontab cleared" + else + echo "$new_crontab" | crontab - + LogInformation "Updated crontab with remaining entries" + fi + LogInformation "Removed $removed_count crontab entries" + else + LogWarning "No matching crontab entries were found to remove" fi } @@ -63,25 +174,48 @@ BackupCrontab() { fi } +ShowUsage() { + echo "Usage: $0 [install|remove|status|help]" + echo "" + echo "Commands:" + echo " install - Install all crontab entries (default)" + echo " remove - Remove all crontab entries" + echo " status - Show status of crontab entries" + echo " help - Show this help message" + echo "" + echo "This script manages crontab entries for monitoring scripts:" + echo "$(printf ' %s\n' "${SCRIPTS[@]}" | sed 's|../metrics/||')" + echo "" + echo "Examples:" + echo " $0 install" + echo " $0 status" +} + Main() { - LogInformation "Starting crontab setup for server management scripts" + local action="${1:-install}" - BackupCrontab - - for script in "${SCRIPTS[@]}"; do - LogInformation "Processing $script..." - - if CheckScriptExists "$script"; then - LogInformation "Script $script found and is executable" - ManageCrontabEntry "$script" - else - LogWarning "Script $script not found or not accessible, skipping..." - fi - done - - LogInformation "Crontab setup completed" - LogInformation "Current crontab entries:" - crontab -l 2>/dev/null | grep -E "(check-disk-usage|check-power-usage|check-temperature)" || LogInformation "No matching entries found" + case "$action" in + install) + LogInformation "Installing crontab entries for server monitoring scripts" + InstallCrontabEntries + LogInformation "Crontab setup completed successfully!" + ;; + remove) + LogInformation "Removing crontab entries for server monitoring scripts" + RemoveCrontabEntries + ;; + status) + ShowStatus + ;; + help|--help|-h) + ShowUsage + ;; + *) + LogError "Unknown action: $action" + ShowUsage + exit 1 + ;; + esac } Main "$@" diff --git a/orchestrator/setup-logstash.sh b/orchestrator/setup-logstash.sh new file mode 100644 index 0000000..0d425a1 --- /dev/null +++ b/orchestrator/setup-logstash.sh @@ -0,0 +1,216 @@ +#!/bin/bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "${SCRIPT_DIR}/../share/logging.sh" + +# Configuration +LOGSTASH_CONFIG_DIR="${SCRIPT_DIR}/../logstash" +LOGSTASH_INSTALL_DIR="/etc/logstash/conf.d" +OUTPUT_CONFIG_FILE="90-output.conf" + +CheckRoot() { + if [[ $EUID -ne 0 ]]; then + LogError "This script must be run as root (use sudo)" + exit 1 + fi +} + +CheckLogstashInstalled() { + if ! command -v logstash &> /dev/null; then + LogWarning "logstash command not found in PATH" + LogWarning "Make sure Logstash is installed and configured" + fi + + if [[ ! -d "$LOGSTASH_INSTALL_DIR" ]]; then + LogInformation "Creating Logstash config directory: $LOGSTASH_INSTALL_DIR" + mkdir -p "$LOGSTASH_INSTALL_DIR" + chmod 755 "$LOGSTASH_INSTALL_DIR" + fi +} + +UpdateOutputConfig() { + local index="$1" + local user="$2" + local password="$3" + local output_file="${LOGSTASH_CONFIG_DIR}/${OUTPUT_CONFIG_FILE}" + local temp_file=$(mktemp) + + if [[ ! -f "$output_file" ]]; then + LogError "Output config file not found: $output_file" + exit 1 + fi + + LogInformation "Updating Elasticsearch configuration in $OUTPUT_CONFIG_FILE" + + sed -e "s/index => \"[^\"]*\"/index => \"$index\"/" \ + -e "s/user => \"[^\"]*\"/user => \"$user\"/" \ + -e "s/password => \"[^\"]*\"/password => \"$password\"/" \ + "$output_file" > "$temp_file" + + if grep -q "index => \"$index\"" "$temp_file" && \ + grep -q "user => \"$user\"" "$temp_file" && \ + grep -q "password => \"$password\"" "$temp_file"; then + mv "$temp_file" "$output_file" + LogInformation "Updated Elasticsearch settings: index=$index, user=$user" + else + rm -f "$temp_file" + LogError "Failed to update Elasticsearch configuration" + exit 1 + fi +} + +InstallLogstashConfigs() { + local installed_count=0 + local failed_count=0 + + LogInformation "Installing Logstash configurations from $LOGSTASH_CONFIG_DIR" + while IFS= read -r -d '' config_file; do + local filename=$(basename "$config_file") + local target_file="${LOGSTASH_INSTALL_DIR}/${filename}" + + LogInformation "Installing $filename..." + if cp "$config_file" "$target_file" && chmod 644 "$target_file"; then + LogInformation "Installed: $target_file" + ((installed_count++)) + else + LogError "Failed to install: $filename" + ((failed_count++)) + fi + done < <(find "$LOGSTASH_CONFIG_DIR" -name "*.conf" -type f -print0) + + LogInformation "Installation summary: $installed_count installed, $failed_count failed" + if [[ $failed_count -gt 0 ]]; then + exit 1 + fi +} + +ValidateLogstashConfig() { + LogInformation "Validating Logstash configuration..." + if command -v logstash &> /dev/null; then + if logstash -t --path.settings=/etc/logstash 2>/dev/null; then + LogInformation "Logstash configuration validation passed" + else + LogWarning "Logstash configuration validation failed or unavailable" + LogWarning "Run manually: sudo logstash -t --path.settings=/etc/logstash" + fi + else + LogWarning "Logstash not available for config validation" + fi +} + +ShowStatus() { + LogInformation "Logstash configuration status:" + LogInformation "" + + if [[ -d "$LOGSTASH_INSTALL_DIR" ]]; then + local config_count=$(find "$LOGSTASH_INSTALL_DIR" -name "*.conf" -type f | wc -l) + LogInformation "Installed configurations: $config_count" + + LogInformation "Configuration files:" + find "$LOGSTASH_INSTALL_DIR" -name "*.conf" -type f -exec basename {} \; | sort | while read -r file; do + LogInformation " ✓ $file" + done + else + LogWarning "Logstash config directory not found: $LOGSTASH_INSTALL_DIR" + fi + + LogInformation "" + LogInformation "Commands:" + LogInformation " Test config: sudo logstash -t --path.settings=/etc/logstash" + LogInformation " Start Logstash: sudo systemctl start logstash" + LogInformation " Check status: sudo systemctl status logstash" +} + +RemoveLogstashConfigs() { + local removed_count=0 + + if [[ ! -d "$LOGSTASH_INSTALL_DIR" ]]; then + LogWarning "Logstash config directory not found: $LOGSTASH_INSTALL_DIR" + return + fi + + LogInformation "Removing Logstash configurations..." + while IFS= read -r -d '' source_file; do + local filename=$(basename "$source_file") + local target_file="${LOGSTASH_INSTALL_DIR}/${filename}" + + if [[ -f "$target_file" ]]; then + LogInformation "Removing $filename..." + rm -f "$target_file" + ((removed_count++)) + fi + done < <(find "$LOGSTASH_CONFIG_DIR" -name "*.conf" -type f -print0) + + if [[ $removed_count -gt 0 ]]; then + LogInformation "Removed $removed_count Logstash configurations" + else + LogWarning "No configurations were found to remove" + fi +} + +ShowUsage() { + echo "Usage: $0 install " + echo " $0 [remove|status|help]" + echo "" + echo "Commands:" + echo " install - Install Logstash configurations with Elasticsearch settings" + echo " remove - Remove all Logstash configurations" + echo " status - Show status of configurations" + echo " help - Show this help message" + echo "" + echo "Parameters for install:" + echo " index - Elasticsearch index name" + echo " user - Elasticsearch username" + echo " password - Elasticsearch password" + echo "" + echo "Examples:" + echo " sudo $0 install myserver-logs elastic mypassword" + echo " sudo $0 status" + echo " sudo $0 remove" +} + +Main() { + local action="$1" + + case "$action" in + install) + if [[ $# -ne 4 ]]; then + LogError "Install command requires 3 parameters: index, user, password" + ShowUsage + exit 1 + fi + + local index="$2" + local user="$3" + local password="$4" + + LogInformation "Installing Logstash configurations" + LogInformation "Elasticsearch settings: index=$index, user=$user" + + CheckRoot + CheckLogstashInstalled + UpdateOutputConfig "$index" "$user" "$password" + InstallLogstashConfigs + ValidateLogstashConfig + LogInformation "Logstash setup completed successfully!" + ;; + remove) + LogInformation "Removing Logstash configurations" + CheckRoot + RemoveLogstashConfigs + ;; + status) + ShowStatus + ;; + help|--help|-h) + ShowUsage + ;; + *) + LogError "Unknown action: $action" + ShowUsage + exit 1 + ;; + esac +} + +Main "$@"