Setup logstash

logstash/10-input.conf

@@ -0,0 +1,31 @@
+input {
+  file {
+    path => "/var/log/temperature.log"
+    start_position => "beginning"
+    sincedb_path => "/dev/null"
+    add_field => {
+        "type" => "temperature"
+        "host.hostname" => "praguetopbox"
+    }
+  }
+
+  file {
+    path => "/var/log/power_usage.log"
+    start_position => "beginning"
+    sincedb_path => "/dev/null"
+    add_field => {
+        "type" => "power"
+        "host.hostname" => "praguetopbox"
+    }
+  }
+
+  file {
+    path => "/var/log/disk_usage.log"
+    start_position => "beginning"
+    sincedb_path => "/dev/null"
+    add_field => {
+        "type" => "disk"
+        "host.hostname" => "praguetopbox"
+    }
+  }
+}

logstash/20-temperature.conf

@@ -0,0 +1,18 @@
+filter {
+  if [type] == "temperature" {
+    dissect {
+      mapping => {
+        "message" => "%{timestamp} %{system.temperature.sensor_name} %{system.temperature.value}°C"
+      }
+    }
+
+    date {
+      match => ["timestamp", "ISO8601"]
+      target => "@timestamp"
+    }
+
+    mutate {
+      convert => { "system.temperature.value" => "float" }
+    }
+  }
+}

logstash/21-power-usage.conf

@@ -0,0 +1,18 @@
+filter {
+  if [type] == "power" {
+    dissect {
+      mapping => {
+        "message" => "%{timestamp} %{system.power.value}"
+      }
+    }
+
+    date {
+      match => ["timestamp", "ISO8601"]
+      target => "@timestamp"
+    }
+
+    mutate {
+      convert => { "system.power.value" => "float" }
+    }
+  }
+}

logstash/22-disk-usage.conf

@@ -0,0 +1,66 @@
+filter {
+  if [type] == "disk" {
+    dissect {
+      mapping => {
+        "message" => "%{timestamp} %{system.disk.mount_point} %{system.disk.used} %{system.disk.total}"
+      }
+    }
+
+    date {
+      match => ["timestamp", "ISO8601"]
+      target => "@timestamp"
+    }
+
+    mutate {
+      convert => { 
+        "system.disk.used" => "integer"
+        "system.disk.total" => "integer"
+      }
+    }
+
+    ruby {
+      code => "
+        used = event.get('system.disk.used').to_f
+        total = event.get('system.disk.total').to_f
+        if total > 0
+          percentage = (used / total) * 100
+          event.set('system.disk.usage_percent', percentage.round(2))
+        end
+      "
+    }
+
+    ruby {
+      code => "
+        def bytes_to_human(bytes)
+          units = ['B', 'KB', 'MB', 'GB', 'TB']
+          return '0 B' if bytes == 0
+          
+          exp = (Math.log(bytes) / Math.log(1024)).floor
+          exp = [exp, units.length - 1].min
+          
+          size = bytes / (1024.0 ** exp)
+          unit = units[exp]
+          
+          if size >= 100
+            '%.0f %s' % [size, unit]
+          elsif size >= 10
+            '%.1f %s' % [size, unit]
+          else
+            '%.2f %s' % [size, unit]
+          end
+        end
+
+        used_bytes = event.get('system.disk.used')
+        total_bytes = event.get('system.disk.total')
+        
+        if used_bytes
+          event.set('system.disk.used_human', bytes_to_human(used_bytes))
+        end
+        
+        if total_bytes
+          event.set('system.disk.total_human', bytes_to_human(total_bytes))
+        end
+      "
+    }
+  }
+}

logstash/90-output.conf

@@ -0,0 +1,12 @@
+output {
+  # Forward logs to Elasticsearch
+  elasticsearch {
+    hosts => ["http://localhost:9200"]
+    index => "replace-index"  # Index pattern based on log type and date
+    user => "replace-user"
+    password => "replace-password"
+  }
+  if "_grokparsefailure" in [tags] {
+    stdout { codec => rubydebug }
+  }
+}

orchestrator/setup-crontab.sh

@@ -3,6 +3,7 @@
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 . "${SCRIPT_DIR}/../share/logging.sh"
 
+# Configuration
 SCRIPTS=(
     "../metrics/disk-usage.sh"
     "../metrics/power-usage.sh" 
@@ -24,6 +25,35 @@ CheckScriptExists() {
     fi
 }
 
+InstallCrontabEntries() {
+    local installed_count=0
+    local failed_count=0
+    
+    BackupCrontab
+    
+    for script in "${SCRIPTS[@]}"; do
+        LogInformation "Processing $script..."
+        
+        if CheckScriptExists "$script"; then
+            LogInformation "Script $script found and is executable"
+            if ManageCrontabEntry "$script"; then
+                ((installed_count++))
+            else
+                ((failed_count++))
+            fi
+        else
+            LogWarning "Script $script not found or not accessible, skipping..."
+            ((failed_count++))
+        fi
+    done
+    
+    LogInformation "Installation summary: $installed_count installed, $failed_count failed"
+    
+    if [[ $failed_count -gt 0 ]]; then
+        exit 1
+    fi
+}
+
 ManageCrontabEntry() {
     local script="$1"
     local script_path="${SCRIPT_DIR}/${script}"
@@ -37,19 +67,100 @@ ManageCrontabEntry() {
         
         if echo "$current_crontab" | grep -q "^\* \* \* \* \* ${script_path}"; then
             LogInformation "Entry for $script is already set to run every minute"
+            return 0
         else
             LogInformation "Updating $script to run every minute"
             local new_crontab
             new_crontab=$(echo "$current_crontab" | grep -v "$script_path")
             new_crontab="${new_crontab}${new_crontab:+$'\n'}${cron_pattern}"
-            echo "$new_crontab" | crontab -
+            if echo "$new_crontab" | crontab -; then
-            LogInformation "Updated crontab entry for $script"
+                LogInformation "Updated crontab entry for $script"
+                return 0
+            else
+                LogError "Failed to update crontab entry for $script"
+                return 1
+            fi
         fi
     else
         LogInformation "Adding new crontab entry for $script"
         local new_crontab="${current_crontab}${current_crontab:+$'\n'}${cron_pattern}"
-        echo "$new_crontab" | crontab -
+        if echo "$new_crontab" | crontab -; then
-        LogInformation "Added crontab entry for $script"
+            LogInformation "Added crontab entry for $script"
+            return 0
+        else
+            LogError "Failed to add crontab entry for $script"
+            return 1
+        fi
+    fi
+}
+
+ShowStatus() {
+    LogInformation "Crontab entries status:"
+    LogInformation ""
+    
+    local found_entries=0
+    for script in "${SCRIPTS[@]}"; do
+        local script_path="${SCRIPT_DIR}/${script}"
+        local script_name=$(basename "$script")
+        
+        if crontab -l 2>/dev/null | grep -q "$script_path"; then
+            LogInformation "✓ $script_name - INSTALLED"
+            ((found_entries++))
+        elif [[ -f "$script_path" ]]; then
+            LogInformation "✗ $script_name - NOT INSTALLED (script available)"
+        else
+            LogInformation "✗ $script_name - SCRIPT MISSING"
+        fi
+    done
+    
+    LogInformation ""
+    if [[ $found_entries -gt 0 ]]; then
+        LogInformation "Commands:"
+        LogInformation "  View crontab: crontab -l"
+        LogInformation "  Edit crontab: crontab -e"
+        LogInformation "  Check logs: sudo tail -f /var/log/syslog | grep CRON"
+    else
+        LogWarning "No crontab entries are currently installed"
+    fi
+}
+
+RemoveCrontabEntries() {
+    local removed_count=0
+    
+    local current_crontab
+    current_crontab=$(crontab -l 2>/dev/null || echo "")
+    
+    if [[ -z "$current_crontab" ]]; then
+        LogWarning "No crontab entries found to remove"
+        return
+    fi
+    
+    local new_crontab="$current_crontab"
+    
+    for script in "${SCRIPTS[@]}"; do
+        local script_path="${SCRIPT_DIR}/${script}"
+        local script_name=$(basename "$script")
+        
+        if echo "$new_crontab" | grep -q "$script_path"; then
+            LogInformation "Removing crontab entry for $script_name..."
+            new_crontab=$(echo "$new_crontab" | grep -v "$script_path")
+            ((removed_count++))
+        else
+            LogDebug "No crontab entry found for $script_name"
+        fi
+    done
+    
+    if [[ $removed_count -gt 0 ]]; then
+        if [[ -z "$new_crontab" ]]; then
+            crontab -r 2>/dev/null || true
+            LogInformation "Removed all entries, crontab cleared"
+        else
+            echo "$new_crontab" | crontab -
+            LogInformation "Updated crontab with remaining entries"
+        fi
+        LogInformation "Removed $removed_count crontab entries"
+    else
+        LogWarning "No matching crontab entries were found to remove"
     fi
 }
 
@@ -63,25 +174,48 @@ BackupCrontab() {
     fi
 }
 
+ShowUsage() {
+    echo "Usage: $0 [install|remove|status|help]"
+    echo ""
+    echo "Commands:"
+    echo "  install  - Install all crontab entries (default)"
+    echo "  remove   - Remove all crontab entries"
+    echo "  status   - Show status of crontab entries"
+    echo "  help     - Show this help message"
+    echo ""
+    echo "This script manages crontab entries for monitoring scripts:"
+    echo "$(printf '  %s\n' "${SCRIPTS[@]}" | sed 's|../metrics/||')"
+    echo ""
+    echo "Examples:"
+    echo "  $0 install"
+    echo "  $0 status"
+}
+
 Main() {
-    LogInformation "Starting crontab setup for server management scripts"
+    local action="${1:-install}"
     
-    BackupCrontab
+    case "$action" in
-    
+        install)
-    for script in "${SCRIPTS[@]}"; do
+            LogInformation "Installing crontab entries for server monitoring scripts"
-        LogInformation "Processing $script..."
+            InstallCrontabEntries
-        
+            LogInformation "Crontab setup completed successfully!"
-        if CheckScriptExists "$script"; then
+            ;;
-            LogInformation "Script $script found and is executable"
+        remove)
-            ManageCrontabEntry "$script"
+            LogInformation "Removing crontab entries for server monitoring scripts"
-        else
+            RemoveCrontabEntries
-            LogWarning "Script $script not found or not accessible, skipping..."
+            ;;
-        fi
+        status)
-    done
+            ShowStatus
-    
+            ;;
-    LogInformation "Crontab setup completed"
+        help|--help|-h)
-    LogInformation "Current crontab entries:"
+            ShowUsage
-    crontab -l 2>/dev/null | grep -E "(check-disk-usage|check-power-usage|check-temperature)" || LogInformation "No matching entries found"
+            ;;
+        *)
+            LogError "Unknown action: $action"
+            ShowUsage
+            exit 1
+            ;;
+    esac
 }
 
 Main "$@"

orchestrator/setup-logstash.sh

@@ -0,0 +1,216 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. "${SCRIPT_DIR}/../share/logging.sh"
+
+# Configuration
+LOGSTASH_CONFIG_DIR="${SCRIPT_DIR}/../logstash"
+LOGSTASH_INSTALL_DIR="/etc/logstash/conf.d"
+OUTPUT_CONFIG_FILE="90-output.conf"
+
+CheckRoot() {
+    if [[ $EUID -ne 0 ]]; then
+        LogError "This script must be run as root (use sudo)"
+        exit 1
+    fi
+}
+
+CheckLogstashInstalled() {
+    if ! command -v logstash &> /dev/null; then
+        LogWarning "logstash command not found in PATH"
+        LogWarning "Make sure Logstash is installed and configured"
+    fi
+    
+    if [[ ! -d "$LOGSTASH_INSTALL_DIR" ]]; then
+        LogInformation "Creating Logstash config directory: $LOGSTASH_INSTALL_DIR"
+        mkdir -p "$LOGSTASH_INSTALL_DIR"
+        chmod 755 "$LOGSTASH_INSTALL_DIR"
+    fi
+}
+
+UpdateOutputConfig() {
+    local index="$1"
+    local user="$2"
+    local password="$3"
+    local output_file="${LOGSTASH_CONFIG_DIR}/${OUTPUT_CONFIG_FILE}"
+    local temp_file=$(mktemp)
+    
+    if [[ ! -f "$output_file" ]]; then
+        LogError "Output config file not found: $output_file"
+        exit 1
+    fi
+    
+    LogInformation "Updating Elasticsearch configuration in $OUTPUT_CONFIG_FILE"
+    
+    sed -e "s/index => \"[^\"]*\"/index => \"$index\"/" \
+        -e "s/user => \"[^\"]*\"/user => \"$user\"/" \
+        -e "s/password => \"[^\"]*\"/password => \"$password\"/" \
+        "$output_file" > "$temp_file"
+    
+    if grep -q "index => \"$index\"" "$temp_file" && \
+       grep -q "user => \"$user\"" "$temp_file" && \
+       grep -q "password => \"$password\"" "$temp_file"; then
+        mv "$temp_file" "$output_file"
+        LogInformation "Updated Elasticsearch settings: index=$index, user=$user"
+    else
+        rm -f "$temp_file"
+        LogError "Failed to update Elasticsearch configuration"
+        exit 1
+    fi
+}
+
+InstallLogstashConfigs() {
+    local installed_count=0
+    local failed_count=0
+    
+    LogInformation "Installing Logstash configurations from $LOGSTASH_CONFIG_DIR"
+    while IFS= read -r -d '' config_file; do
+        local filename=$(basename "$config_file")
+        local target_file="${LOGSTASH_INSTALL_DIR}/${filename}"
+        
+        LogInformation "Installing $filename..."
+        if cp "$config_file" "$target_file" && chmod 644 "$target_file"; then
+            LogInformation "Installed: $target_file"
+            ((installed_count++))
+        else
+            LogError "Failed to install: $filename"
+            ((failed_count++))
+        fi
+    done < <(find "$LOGSTASH_CONFIG_DIR" -name "*.conf" -type f -print0)
+    
+    LogInformation "Installation summary: $installed_count installed, $failed_count failed"
+    if [[ $failed_count -gt 0 ]]; then
+        exit 1
+    fi
+}
+
+ValidateLogstashConfig() {
+    LogInformation "Validating Logstash configuration..."
+    if command -v logstash &> /dev/null; then
+        if logstash -t --path.settings=/etc/logstash 2>/dev/null; then
+            LogInformation "Logstash configuration validation passed"
+        else
+            LogWarning "Logstash configuration validation failed or unavailable"
+            LogWarning "Run manually: sudo logstash -t --path.settings=/etc/logstash"
+        fi
+    else
+        LogWarning "Logstash not available for config validation"
+    fi
+}
+
+ShowStatus() {
+    LogInformation "Logstash configuration status:"
+    LogInformation ""
+    
+    if [[ -d "$LOGSTASH_INSTALL_DIR" ]]; then
+        local config_count=$(find "$LOGSTASH_INSTALL_DIR" -name "*.conf" -type f | wc -l)
+        LogInformation "Installed configurations: $config_count"
+        
+        LogInformation "Configuration files:"
+        find "$LOGSTASH_INSTALL_DIR" -name "*.conf" -type f -exec basename {} \; | sort | while read -r file; do
+            LogInformation "  ✓ $file"
+        done
+    else
+        LogWarning "Logstash config directory not found: $LOGSTASH_INSTALL_DIR"
+    fi
+    
+    LogInformation ""
+    LogInformation "Commands:"
+    LogInformation "  Test config: sudo logstash -t --path.settings=/etc/logstash"
+    LogInformation "  Start Logstash: sudo systemctl start logstash"
+    LogInformation "  Check status: sudo systemctl status logstash"
+}
+
+RemoveLogstashConfigs() {
+    local removed_count=0
+    
+    if [[ ! -d "$LOGSTASH_INSTALL_DIR" ]]; then
+        LogWarning "Logstash config directory not found: $LOGSTASH_INSTALL_DIR"
+        return
+    fi
+    
+    LogInformation "Removing Logstash configurations..."
+    while IFS= read -r -d '' source_file; do
+        local filename=$(basename "$source_file")
+        local target_file="${LOGSTASH_INSTALL_DIR}/${filename}"
+        
+        if [[ -f "$target_file" ]]; then
+            LogInformation "Removing $filename..."
+            rm -f "$target_file"
+            ((removed_count++))
+        fi
+    done < <(find "$LOGSTASH_CONFIG_DIR" -name "*.conf" -type f -print0)
+    
+    if [[ $removed_count -gt 0 ]]; then
+        LogInformation "Removed $removed_count Logstash configurations"
+    else
+        LogWarning "No configurations were found to remove"
+    fi
+}
+
+ShowUsage() {
+    echo "Usage: $0 install <index> <user> <password>"
+    echo "       $0 [remove|status|help]"
+    echo ""
+    echo "Commands:"
+    echo "  install  - Install Logstash configurations with Elasticsearch settings"
+    echo "  remove   - Remove all Logstash configurations"
+    echo "  status   - Show status of configurations"
+    echo "  help     - Show this help message"
+    echo ""
+    echo "Parameters for install:"
+    echo "  index    - Elasticsearch index name"
+    echo "  user     - Elasticsearch username"
+    echo "  password - Elasticsearch password"
+    echo ""
+    echo "Examples:"
+    echo "  sudo $0 install myserver-logs elastic mypassword"
+    echo "  sudo $0 status"
+    echo "  sudo $0 remove"
+}
+
+Main() {
+    local action="$1"
+    
+    case "$action" in
+        install)
+            if [[ $# -ne 4 ]]; then
+                LogError "Install command requires 3 parameters: index, user, password"
+                ShowUsage
+                exit 1
+            fi
+            
+            local index="$2"
+            local user="$3"
+            local password="$4"
+            
+            LogInformation "Installing Logstash configurations"
+            LogInformation "Elasticsearch settings: index=$index, user=$user"
+            
+            CheckRoot
+            CheckLogstashInstalled
+            UpdateOutputConfig "$index" "$user" "$password"
+            InstallLogstashConfigs
+            ValidateLogstashConfig
+            LogInformation "Logstash setup completed successfully!"
+            ;;
+        remove)
+            LogInformation "Removing Logstash configurations"
+            CheckRoot
+            RemoveLogstashConfigs
+            ;;
+        status)
+            ShowStatus
+            ;;
+        help|--help|-h)
+            ShowUsage
+            ;;
+        *)
+            LogError "Unknown action: $action"
+            ShowUsage
+            exit 1
+            ;;
+    esac
+}
+
+Main "$@"