1. System Resource Monitoring
1.1 CPU Issue Diagnosis
# Check CPU usage
top
htop
# Sort processes by CPU usage
ps aux --sort=-%cpu | head
# Check CPU information
lscpu
cat /proc/cpuinfo
# Monitor CPU usage
mpstat -P ALL 1
# Check interrupt statistics
cat /proc/interrupts
1.2 Memory Issue Diagnosis
# Check memory usage
free -h
cat /proc/meminfo
# Check processes using the most memory
ps aux --sort=-%mem | head
# Monitor memory usage
vmstat 1
# Check slab memory usage
slabtop
# Check for memory leaks
valgrind --leak-check=full [program_name]
1.3 Disk Issue Diagnosis
# Check disk space
df -h
df -i # Check inode usage
# Check large files/directories
du -sh /* | sort -rh | head
du -ah /path/to/dir | sort -rh | head
# Monitor disk IO
iostat -x 1
iotop
# Check filesystem errors
fsck /dev/sda1
# Check disk smart information
smartctl -a /dev/sda
2. Network Troubleshooting
2.1 Basic Network Checks
# Check network interfaces
ip addr show
ifconfig
# Check routing table
ip route show
route -n
# Test network connectivity
ping [target_IP_or_domain]
ping6 [IPv6_address]
# Trace route path
traceroute [target_IP_or_domain]
mtr [target_IP_or_domain]
2.2 Port and Service Checks
# Check listening ports
netstat -tulpn
ss -tulpn
# Check specific port
telnet [IP] [port]
nc -zv [IP] [port]
# Check firewall rules
iptables -L -n
firewall-cmd --list-all # CentOS/RHEL
ufw status # Ubuntu
# Check DNS resolution
nslookup [domain]
dig [domain]
2.3 Network Performance Analysis
# Check network connection status
netstat -an | grep :80 | wc -l # Count HTTP connections
# Monitor network traffic
iftop
nethogs
# Packet capture analysis
tcpdump -i [interface] -w capture.pcap
tcpdump -i [interface] port 80 # Capture traffic on port 80
# Bandwidth test
iperf3 -c [server_IP] # Client
iperf3 -s # Server
3. System Service Failures
3.1 Service Status Check
# Systemd system
systemctl status [service_name]
systemctl is-active [service_name]
systemctl is-enabled [service_name]
# SysVinit system
service [service_name] status
chkconfig --list [service_name]
# Check service logs
journalctl -u [service_name] -f
tail -f /var/log/[service_log_file]
3.2 Process Management
# Find process
ps aux | grep [process_name]
pgrep [process_name]
# View process tree
pstree -p
ps -ef --forest
# Kill process
kill [PID]
kill -9 [PID] # Force kill
pkill [process_name]
# Check files opened by process
lsof -p [PID]
lsof -i :80 # Check process using port 80
4. Log Analysis
4.1 System Log Viewing
# View system logs
tail -f /var/log/messages # CentOS/RHEL
tail -f /var/log/syslog # Ubuntu/Debian
# View kernel logs
dmesg
dmesg | grep -i error
# View login logs
last
lastlog
# View security logs
tail -f /var/log/secure # CentOS/RHEL
tail -f /var/log/auth.log # Ubuntu/Debian
4.2 Log Analysis Techniques
# Search for error messages
grep -i error /var/log/messages
grep -r "error" /var/log/
# Count occurrences in logs
grep "Failed password" /var/log/secure | wc -l
# Logs within a time range
sed -n '/start_time/,/end_time/p' /var/log/messages
# Real-time log monitoring
tail -f /var/log/apache2/access.log | grep "404"
5. Hardware Failure Diagnosis
5.1 Hardware Information Check
# Check hardware information
lshw
dmidecode
# Check USB devices
lsusb
lsusb -v
# Check PCI devices
lspci
lspci -v
# Check block devices
lsblk
blkid
5.2 Hardware Status Monitoring
# Check CPU temperature
sensors
# Check hard drive temperature
hddtemp /dev/sda
# Monitor hardware errors
dmesg | grep -i "hardware error"
cat /var/log/messages | grep -i "hardware error"
6. Performance Optimization and Tuning
6.1 System Performance Analysis
# Overall system performance monitoring
sar -u 13 # CPU usage
sar -r 13 # Memory usage
sar -b 13 # IO usage
sar -n DEV 13 # Network usage
# Generate system status report
sysstat
# Analyze system bottlenecks
perf record -g [command]
perf report
6.2 Kernel Parameter Tuning
# View current kernel parameters
sysctl -a
# Temporarily modify parameters
sysctl -w kernel.parameter=value
# Permanently modify
echo "kernel.parameter=value" >> /etc/sysctl.conf
sysctl -p
7. Common Failure Scenarios Handling
Scenario 1: System Response is Slow
# Quick diagnosis steps
top # Check CPU and memory usage
iostat -x 1 # Check disk IO
dmesg | tail # Check kernel errors
netstat -an | grep :80 | wc -l # Check number of network connections
Scenario 2: Insufficient Disk Space
# Cleanup steps
df -h # Confirm problematic partition
du -sh /* | sort -rh | head # Find large files
# Clean cache, logs, temporary files
rm -rf /tmp/*
journalctl --vacuum-size=100M # Clean system logs
Scenario 3: Service Cannot Start
# Troubleshooting steps
systemctl status [service_name] # Check service status
journalctl -u [service_name] -f # Check service logs
ss -tulpn | grep [port] # Check port occupation
ps aux | grep [process_name] # Check if process exists
Scenario 4: Network Connection Issues
# Troubleshooting steps
ping [gateway] # Check LAN connectivity
ping 8.8.8.8 # Check external network connectivity
nslookup google.com # Check DNS resolution
traceroute google.com # Check route path
iptables -L -n # Check firewall rules
