mirror of
https://github.com/AlexMacocian/ServerManagementUtils.git
synced 2026-07-15 15:19:58 +00:00
Setup logstash
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
input {
|
||||
file {
|
||||
path => "/var/log/temperature.log"
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/dev/null"
|
||||
add_field => {
|
||||
"type" => "temperature"
|
||||
"host.hostname" => "praguetopbox"
|
||||
}
|
||||
}
|
||||
|
||||
file {
|
||||
path => "/var/log/power_usage.log"
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/dev/null"
|
||||
add_field => {
|
||||
"type" => "power"
|
||||
"host.hostname" => "praguetopbox"
|
||||
}
|
||||
}
|
||||
|
||||
file {
|
||||
path => "/var/log/disk_usage.log"
|
||||
start_position => "beginning"
|
||||
sincedb_path => "/dev/null"
|
||||
add_field => {
|
||||
"type" => "disk"
|
||||
"host.hostname" => "praguetopbox"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
filter {
|
||||
if [type] == "temperature" {
|
||||
dissect {
|
||||
mapping => {
|
||||
"message" => "%{timestamp} %{system.temperature.sensor_name} %{system.temperature.value}°C"
|
||||
}
|
||||
}
|
||||
|
||||
date {
|
||||
match => ["timestamp", "ISO8601"]
|
||||
target => "@timestamp"
|
||||
}
|
||||
|
||||
mutate {
|
||||
convert => { "system.temperature.value" => "float" }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
filter {
|
||||
if [type] == "power" {
|
||||
dissect {
|
||||
mapping => {
|
||||
"message" => "%{timestamp} %{system.power.value}"
|
||||
}
|
||||
}
|
||||
|
||||
date {
|
||||
match => ["timestamp", "ISO8601"]
|
||||
target => "@timestamp"
|
||||
}
|
||||
|
||||
mutate {
|
||||
convert => { "system.power.value" => "float" }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
filter {
|
||||
if [type] == "disk" {
|
||||
dissect {
|
||||
mapping => {
|
||||
"message" => "%{timestamp} %{system.disk.mount_point} %{system.disk.used} %{system.disk.total}"
|
||||
}
|
||||
}
|
||||
|
||||
date {
|
||||
match => ["timestamp", "ISO8601"]
|
||||
target => "@timestamp"
|
||||
}
|
||||
|
||||
mutate {
|
||||
convert => {
|
||||
"system.disk.used" => "integer"
|
||||
"system.disk.total" => "integer"
|
||||
}
|
||||
}
|
||||
|
||||
ruby {
|
||||
code => "
|
||||
used = event.get('system.disk.used').to_f
|
||||
total = event.get('system.disk.total').to_f
|
||||
if total > 0
|
||||
percentage = (used / total) * 100
|
||||
event.set('system.disk.usage_percent', percentage.round(2))
|
||||
end
|
||||
"
|
||||
}
|
||||
|
||||
ruby {
|
||||
code => "
|
||||
def bytes_to_human(bytes)
|
||||
units = ['B', 'KB', 'MB', 'GB', 'TB']
|
||||
return '0 B' if bytes == 0
|
||||
|
||||
exp = (Math.log(bytes) / Math.log(1024)).floor
|
||||
exp = [exp, units.length - 1].min
|
||||
|
||||
size = bytes / (1024.0 ** exp)
|
||||
unit = units[exp]
|
||||
|
||||
if size >= 100
|
||||
'%.0f %s' % [size, unit]
|
||||
elsif size >= 10
|
||||
'%.1f %s' % [size, unit]
|
||||
else
|
||||
'%.2f %s' % [size, unit]
|
||||
end
|
||||
end
|
||||
|
||||
used_bytes = event.get('system.disk.used')
|
||||
total_bytes = event.get('system.disk.total')
|
||||
|
||||
if used_bytes
|
||||
event.set('system.disk.used_human', bytes_to_human(used_bytes))
|
||||
end
|
||||
|
||||
if total_bytes
|
||||
event.set('system.disk.total_human', bytes_to_human(total_bytes))
|
||||
end
|
||||
"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
output {
|
||||
# Forward logs to Elasticsearch
|
||||
elasticsearch {
|
||||
hosts => ["http://localhost:9200"]
|
||||
index => "replace-index" # Index pattern based on log type and date
|
||||
user => "replace-user"
|
||||
password => "replace-password"
|
||||
}
|
||||
if "_grokparsefailure" in [tags] {
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
}
|
||||
+155
-21
@@ -3,6 +3,7 @@
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
. "${SCRIPT_DIR}/../share/logging.sh"
|
||||
|
||||
# Configuration
|
||||
SCRIPTS=(
|
||||
"../metrics/disk-usage.sh"
|
||||
"../metrics/power-usage.sh"
|
||||
@@ -24,6 +25,35 @@ CheckScriptExists() {
|
||||
fi
|
||||
}
|
||||
|
||||
InstallCrontabEntries() {
|
||||
local installed_count=0
|
||||
local failed_count=0
|
||||
|
||||
BackupCrontab
|
||||
|
||||
for script in "${SCRIPTS[@]}"; do
|
||||
LogInformation "Processing $script..."
|
||||
|
||||
if CheckScriptExists "$script"; then
|
||||
LogInformation "Script $script found and is executable"
|
||||
if ManageCrontabEntry "$script"; then
|
||||
((installed_count++))
|
||||
else
|
||||
((failed_count++))
|
||||
fi
|
||||
else
|
||||
LogWarning "Script $script not found or not accessible, skipping..."
|
||||
((failed_count++))
|
||||
fi
|
||||
done
|
||||
|
||||
LogInformation "Installation summary: $installed_count installed, $failed_count failed"
|
||||
|
||||
if [[ $failed_count -gt 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
ManageCrontabEntry() {
|
||||
local script="$1"
|
||||
local script_path="${SCRIPT_DIR}/${script}"
|
||||
@@ -37,19 +67,100 @@ ManageCrontabEntry() {
|
||||
|
||||
if echo "$current_crontab" | grep -q "^\* \* \* \* \* ${script_path}"; then
|
||||
LogInformation "Entry for $script is already set to run every minute"
|
||||
return 0
|
||||
else
|
||||
LogInformation "Updating $script to run every minute"
|
||||
local new_crontab
|
||||
new_crontab=$(echo "$current_crontab" | grep -v "$script_path")
|
||||
new_crontab="${new_crontab}${new_crontab:+$'\n'}${cron_pattern}"
|
||||
echo "$new_crontab" | crontab -
|
||||
LogInformation "Updated crontab entry for $script"
|
||||
if echo "$new_crontab" | crontab -; then
|
||||
LogInformation "Updated crontab entry for $script"
|
||||
return 0
|
||||
else
|
||||
LogError "Failed to update crontab entry for $script"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
LogInformation "Adding new crontab entry for $script"
|
||||
local new_crontab="${current_crontab}${current_crontab:+$'\n'}${cron_pattern}"
|
||||
echo "$new_crontab" | crontab -
|
||||
LogInformation "Added crontab entry for $script"
|
||||
if echo "$new_crontab" | crontab -; then
|
||||
LogInformation "Added crontab entry for $script"
|
||||
return 0
|
||||
else
|
||||
LogError "Failed to add crontab entry for $script"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
ShowStatus() {
|
||||
LogInformation "Crontab entries status:"
|
||||
LogInformation ""
|
||||
|
||||
local found_entries=0
|
||||
for script in "${SCRIPTS[@]}"; do
|
||||
local script_path="${SCRIPT_DIR}/${script}"
|
||||
local script_name=$(basename "$script")
|
||||
|
||||
if crontab -l 2>/dev/null | grep -q "$script_path"; then
|
||||
LogInformation "✓ $script_name - INSTALLED"
|
||||
((found_entries++))
|
||||
elif [[ -f "$script_path" ]]; then
|
||||
LogInformation "✗ $script_name - NOT INSTALLED (script available)"
|
||||
else
|
||||
LogInformation "✗ $script_name - SCRIPT MISSING"
|
||||
fi
|
||||
done
|
||||
|
||||
LogInformation ""
|
||||
if [[ $found_entries -gt 0 ]]; then
|
||||
LogInformation "Commands:"
|
||||
LogInformation " View crontab: crontab -l"
|
||||
LogInformation " Edit crontab: crontab -e"
|
||||
LogInformation " Check logs: sudo tail -f /var/log/syslog | grep CRON"
|
||||
else
|
||||
LogWarning "No crontab entries are currently installed"
|
||||
fi
|
||||
}
|
||||
|
||||
RemoveCrontabEntries() {
|
||||
local removed_count=0
|
||||
|
||||
local current_crontab
|
||||
current_crontab=$(crontab -l 2>/dev/null || echo "")
|
||||
|
||||
if [[ -z "$current_crontab" ]]; then
|
||||
LogWarning "No crontab entries found to remove"
|
||||
return
|
||||
fi
|
||||
|
||||
local new_crontab="$current_crontab"
|
||||
|
||||
for script in "${SCRIPTS[@]}"; do
|
||||
local script_path="${SCRIPT_DIR}/${script}"
|
||||
local script_name=$(basename "$script")
|
||||
|
||||
if echo "$new_crontab" | grep -q "$script_path"; then
|
||||
LogInformation "Removing crontab entry for $script_name..."
|
||||
new_crontab=$(echo "$new_crontab" | grep -v "$script_path")
|
||||
((removed_count++))
|
||||
else
|
||||
LogDebug "No crontab entry found for $script_name"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $removed_count -gt 0 ]]; then
|
||||
if [[ -z "$new_crontab" ]]; then
|
||||
crontab -r 2>/dev/null || true
|
||||
LogInformation "Removed all entries, crontab cleared"
|
||||
else
|
||||
echo "$new_crontab" | crontab -
|
||||
LogInformation "Updated crontab with remaining entries"
|
||||
fi
|
||||
LogInformation "Removed $removed_count crontab entries"
|
||||
else
|
||||
LogWarning "No matching crontab entries were found to remove"
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -63,25 +174,48 @@ BackupCrontab() {
|
||||
fi
|
||||
}
|
||||
|
||||
ShowUsage() {
|
||||
echo "Usage: $0 [install|remove|status|help]"
|
||||
echo ""
|
||||
echo "Commands:"
|
||||
echo " install - Install all crontab entries (default)"
|
||||
echo " remove - Remove all crontab entries"
|
||||
echo " status - Show status of crontab entries"
|
||||
echo " help - Show this help message"
|
||||
echo ""
|
||||
echo "This script manages crontab entries for monitoring scripts:"
|
||||
echo "$(printf ' %s\n' "${SCRIPTS[@]}" | sed 's|../metrics/||')"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 install"
|
||||
echo " $0 status"
|
||||
}
|
||||
|
||||
Main() {
|
||||
LogInformation "Starting crontab setup for server management scripts"
|
||||
local action="${1:-install}"
|
||||
|
||||
BackupCrontab
|
||||
|
||||
for script in "${SCRIPTS[@]}"; do
|
||||
LogInformation "Processing $script..."
|
||||
|
||||
if CheckScriptExists "$script"; then
|
||||
LogInformation "Script $script found and is executable"
|
||||
ManageCrontabEntry "$script"
|
||||
else
|
||||
LogWarning "Script $script not found or not accessible, skipping..."
|
||||
fi
|
||||
done
|
||||
|
||||
LogInformation "Crontab setup completed"
|
||||
LogInformation "Current crontab entries:"
|
||||
crontab -l 2>/dev/null | grep -E "(check-disk-usage|check-power-usage|check-temperature)" || LogInformation "No matching entries found"
|
||||
case "$action" in
|
||||
install)
|
||||
LogInformation "Installing crontab entries for server monitoring scripts"
|
||||
InstallCrontabEntries
|
||||
LogInformation "Crontab setup completed successfully!"
|
||||
;;
|
||||
remove)
|
||||
LogInformation "Removing crontab entries for server monitoring scripts"
|
||||
RemoveCrontabEntries
|
||||
;;
|
||||
status)
|
||||
ShowStatus
|
||||
;;
|
||||
help|--help|-h)
|
||||
ShowUsage
|
||||
;;
|
||||
*)
|
||||
LogError "Unknown action: $action"
|
||||
ShowUsage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
Main "$@"
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
#!/bin/bash
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
. "${SCRIPT_DIR}/../share/logging.sh"
|
||||
|
||||
# Configuration
|
||||
LOGSTASH_CONFIG_DIR="${SCRIPT_DIR}/../logstash"
|
||||
LOGSTASH_INSTALL_DIR="/etc/logstash/conf.d"
|
||||
OUTPUT_CONFIG_FILE="90-output.conf"
|
||||
|
||||
CheckRoot() {
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
LogError "This script must be run as root (use sudo)"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
CheckLogstashInstalled() {
|
||||
if ! command -v logstash &> /dev/null; then
|
||||
LogWarning "logstash command not found in PATH"
|
||||
LogWarning "Make sure Logstash is installed and configured"
|
||||
fi
|
||||
|
||||
if [[ ! -d "$LOGSTASH_INSTALL_DIR" ]]; then
|
||||
LogInformation "Creating Logstash config directory: $LOGSTASH_INSTALL_DIR"
|
||||
mkdir -p "$LOGSTASH_INSTALL_DIR"
|
||||
chmod 755 "$LOGSTASH_INSTALL_DIR"
|
||||
fi
|
||||
}
|
||||
|
||||
UpdateOutputConfig() {
|
||||
local index="$1"
|
||||
local user="$2"
|
||||
local password="$3"
|
||||
local output_file="${LOGSTASH_CONFIG_DIR}/${OUTPUT_CONFIG_FILE}"
|
||||
local temp_file=$(mktemp)
|
||||
|
||||
if [[ ! -f "$output_file" ]]; then
|
||||
LogError "Output config file not found: $output_file"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LogInformation "Updating Elasticsearch configuration in $OUTPUT_CONFIG_FILE"
|
||||
|
||||
sed -e "s/index => \"[^\"]*\"/index => \"$index\"/" \
|
||||
-e "s/user => \"[^\"]*\"/user => \"$user\"/" \
|
||||
-e "s/password => \"[^\"]*\"/password => \"$password\"/" \
|
||||
"$output_file" > "$temp_file"
|
||||
|
||||
if grep -q "index => \"$index\"" "$temp_file" && \
|
||||
grep -q "user => \"$user\"" "$temp_file" && \
|
||||
grep -q "password => \"$password\"" "$temp_file"; then
|
||||
mv "$temp_file" "$output_file"
|
||||
LogInformation "Updated Elasticsearch settings: index=$index, user=$user"
|
||||
else
|
||||
rm -f "$temp_file"
|
||||
LogError "Failed to update Elasticsearch configuration"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
InstallLogstashConfigs() {
|
||||
local installed_count=0
|
||||
local failed_count=0
|
||||
|
||||
LogInformation "Installing Logstash configurations from $LOGSTASH_CONFIG_DIR"
|
||||
while IFS= read -r -d '' config_file; do
|
||||
local filename=$(basename "$config_file")
|
||||
local target_file="${LOGSTASH_INSTALL_DIR}/${filename}"
|
||||
|
||||
LogInformation "Installing $filename..."
|
||||
if cp "$config_file" "$target_file" && chmod 644 "$target_file"; then
|
||||
LogInformation "Installed: $target_file"
|
||||
((installed_count++))
|
||||
else
|
||||
LogError "Failed to install: $filename"
|
||||
((failed_count++))
|
||||
fi
|
||||
done < <(find "$LOGSTASH_CONFIG_DIR" -name "*.conf" -type f -print0)
|
||||
|
||||
LogInformation "Installation summary: $installed_count installed, $failed_count failed"
|
||||
if [[ $failed_count -gt 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
ValidateLogstashConfig() {
|
||||
LogInformation "Validating Logstash configuration..."
|
||||
if command -v logstash &> /dev/null; then
|
||||
if logstash -t --path.settings=/etc/logstash 2>/dev/null; then
|
||||
LogInformation "Logstash configuration validation passed"
|
||||
else
|
||||
LogWarning "Logstash configuration validation failed or unavailable"
|
||||
LogWarning "Run manually: sudo logstash -t --path.settings=/etc/logstash"
|
||||
fi
|
||||
else
|
||||
LogWarning "Logstash not available for config validation"
|
||||
fi
|
||||
}
|
||||
|
||||
ShowStatus() {
|
||||
LogInformation "Logstash configuration status:"
|
||||
LogInformation ""
|
||||
|
||||
if [[ -d "$LOGSTASH_INSTALL_DIR" ]]; then
|
||||
local config_count=$(find "$LOGSTASH_INSTALL_DIR" -name "*.conf" -type f | wc -l)
|
||||
LogInformation "Installed configurations: $config_count"
|
||||
|
||||
LogInformation "Configuration files:"
|
||||
find "$LOGSTASH_INSTALL_DIR" -name "*.conf" -type f -exec basename {} \; | sort | while read -r file; do
|
||||
LogInformation " ✓ $file"
|
||||
done
|
||||
else
|
||||
LogWarning "Logstash config directory not found: $LOGSTASH_INSTALL_DIR"
|
||||
fi
|
||||
|
||||
LogInformation ""
|
||||
LogInformation "Commands:"
|
||||
LogInformation " Test config: sudo logstash -t --path.settings=/etc/logstash"
|
||||
LogInformation " Start Logstash: sudo systemctl start logstash"
|
||||
LogInformation " Check status: sudo systemctl status logstash"
|
||||
}
|
||||
|
||||
RemoveLogstashConfigs() {
|
||||
local removed_count=0
|
||||
|
||||
if [[ ! -d "$LOGSTASH_INSTALL_DIR" ]]; then
|
||||
LogWarning "Logstash config directory not found: $LOGSTASH_INSTALL_DIR"
|
||||
return
|
||||
fi
|
||||
|
||||
LogInformation "Removing Logstash configurations..."
|
||||
while IFS= read -r -d '' source_file; do
|
||||
local filename=$(basename "$source_file")
|
||||
local target_file="${LOGSTASH_INSTALL_DIR}/${filename}"
|
||||
|
||||
if [[ -f "$target_file" ]]; then
|
||||
LogInformation "Removing $filename..."
|
||||
rm -f "$target_file"
|
||||
((removed_count++))
|
||||
fi
|
||||
done < <(find "$LOGSTASH_CONFIG_DIR" -name "*.conf" -type f -print0)
|
||||
|
||||
if [[ $removed_count -gt 0 ]]; then
|
||||
LogInformation "Removed $removed_count Logstash configurations"
|
||||
else
|
||||
LogWarning "No configurations were found to remove"
|
||||
fi
|
||||
}
|
||||
|
||||
ShowUsage() {
|
||||
echo "Usage: $0 install <index> <user> <password>"
|
||||
echo " $0 [remove|status|help]"
|
||||
echo ""
|
||||
echo "Commands:"
|
||||
echo " install - Install Logstash configurations with Elasticsearch settings"
|
||||
echo " remove - Remove all Logstash configurations"
|
||||
echo " status - Show status of configurations"
|
||||
echo " help - Show this help message"
|
||||
echo ""
|
||||
echo "Parameters for install:"
|
||||
echo " index - Elasticsearch index name"
|
||||
echo " user - Elasticsearch username"
|
||||
echo " password - Elasticsearch password"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " sudo $0 install myserver-logs elastic mypassword"
|
||||
echo " sudo $0 status"
|
||||
echo " sudo $0 remove"
|
||||
}
|
||||
|
||||
Main() {
|
||||
local action="$1"
|
||||
|
||||
case "$action" in
|
||||
install)
|
||||
if [[ $# -ne 4 ]]; then
|
||||
LogError "Install command requires 3 parameters: index, user, password"
|
||||
ShowUsage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local index="$2"
|
||||
local user="$3"
|
||||
local password="$4"
|
||||
|
||||
LogInformation "Installing Logstash configurations"
|
||||
LogInformation "Elasticsearch settings: index=$index, user=$user"
|
||||
|
||||
CheckRoot
|
||||
CheckLogstashInstalled
|
||||
UpdateOutputConfig "$index" "$user" "$password"
|
||||
InstallLogstashConfigs
|
||||
ValidateLogstashConfig
|
||||
LogInformation "Logstash setup completed successfully!"
|
||||
;;
|
||||
remove)
|
||||
LogInformation "Removing Logstash configurations"
|
||||
CheckRoot
|
||||
RemoveLogstashConfigs
|
||||
;;
|
||||
status)
|
||||
ShowStatus
|
||||
;;
|
||||
help|--help|-h)
|
||||
ShowUsage
|
||||
;;
|
||||
*)
|
||||
LogError "Unknown action: $action"
|
||||
ShowUsage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
Main "$@"
|
||||
Reference in New Issue
Block a user