Grafana Breach: Coinbase Cartel Attack — Detection, Investigation, and Hardening Guide

---
title: Grafana Unusual Administrative Actions
id: a7f4b2e1-8c9d-4a3f-9e5c-1d2a3b4c5d6e
status: experimental
description: Detects suspicious administrative activity in Grafana, such as bulk user creation or unexpected configuration changes
references:
  - https://grafana.com/docs/grafana/latest/administration/
author: Security Arsenal
date: 2025/03/26
tags:
  - attack.persistence
  - attack.t1098
logsource:
 category: webserver
 product: grafana
detection:
 selection:
 cs-method|contains:
   - 'POST'
 cs-uri-stem|contains:
   - '/api/users'
   - '/api/admin/users'
   - '/api/datasources'
   - '/api/dashboards/uid'
   - '/api/teams'
 filter:
 sc-status: 200
 condition: selection and filter
falsepositives:
  - Legitimate administrative management
level: high
---
title: Grafana Bulk Data Export Activity
id: b8c5d3f2-9d0e-4b4g-0f6d-2e3b4c5d6e7f
status: experimental
description: Detects potential data exfiltration through bulk dashboard or data exports from Grafana
references:
  - https://grafana.com/docs/grafana/latest/http_api/
author: Security Arsenal
date: 2025/03/26
tags:
  - attack.exfiltration
  - attack.t1041
logsource:
 category: webserver
 product: grafana
detection:
 selection:
 cs-uri-stem|contains:
   - '/api/dashboards/uid'
   - '/api/datasources'
   - '/api/snapshots'
   - '/api/search'
 cs-method|contains:
   - 'GET'
   - 'POST'
 filter:
 cs-uri-query|contains:
   - 'export'
   - 'download'
 condition: selection and filter
falsepositives:
  - Legitimate dashboard backups or migrations
level: medium
---
title: Grafana Authentication Anomalies
id: c9d6e4g3-0e1f-5c5h-1g7e-3f4c5d6e7f8g
status: experimental
description: Detects unusual authentication patterns including multiple failed attempts or successful logins from new geographic locations
references:
  - https://grafana.com/docs/grafana/latest/HTTP_API/
author: Security Arsenal
date: 2025/03/26
tags:
  - attack.initial_access
  - attack.t1078
logsource:
 category: authentication
 product: grafana
detection:
 selection:
 eventSource: 'grafana'
 eventName|contains:
   - 'login'
   - 'auth'
 filter:
 result|contains:
   - 'success'
 condition: selection and filter
 timeframe: 5m
falsepositives:
  - Legitimate user logins from new locations
level: medium

// Hunt for unusual Grafana administrative activity
// Works with Syslog or CommonSecurityLog formats
let grafana_events = Syslog
| where SyslogMessage contains "grafana" or ProcessName contains "grafana";
grafana_events
| where SyslogMessage has_all ("POST", "/api/users") or 
       SyslogMessage has_all ("POST", "/api/admin/users") or
       SyslogMessage has_all ("POST", "/api/datasources")
| extend EventTime = TimeGenerated
| extend SourceIP = IPAddress
| extend Details = SyslogMessage
| summarize Count = count() by SourceIP, EventTime, Details
| where Count > 3  // Threshold for bulk actions
| order by EventTime desc
| project EventTime, SourceIP, Count, Details


// Identify potential Grafana data exfiltration activity
let grafana_exfil = DeviceNetworkEvents
| where RemoteUrl contains "grafana" or InitiatingProcessName contains "grafana";
grafana_exfil
| where RemoteUrl has_any ("/api/dashboards/uid", "/api/snapshots", "/api/export")
| extend ExfilTime = Timestamp
| extend SourceDevice = DeviceName
| extend RemoteEndpoint = RemoteUrl
| summarize TotalBytes = sum(SentBytes) by SourceDevice, RemoteEndpoint, ExfilTime
| where TotalBytes > 500000  // Threshold for significant data transfers
| order by ExfilTime desc
| project ExfilTime, SourceDevice, RemoteEndpoint, TotalBytes


// Detect Grafana authentication anomalies
let grafana_auth = SigninLogs
| extend AppDisplayName = coalesce(AppDisplayName, ApplicationDisplayName)
| where AppDisplayName contains "Grafana";
grafana_auth
| extend LocationInfo = strcat(Location, " | ", City, ", ", State)
| extend AuthenticationMethod = coalesce(AuthenticationMethod, AuthenticationRequirement)
| summarize FirstSeen = min(TimeGenerated), LastSeen = max(TimeGenerated), AttemptCount = count() by 
    UserPrincipalName, IPAddress, LocationInfo, AuthenticationMethod, Result
| where Result == "success"
| where AttemptCount < 5  // Filter out frequent, legitimate users
| project FirstSeen, LastSeen, UserPrincipalName, IPAddress, LocationInfo, AttemptCount, Result
| order by LastSeen desc

-- Hunt for suspicious Grafana configuration files and potential indicators of compromise
SELECT FullPath, Size, Mtime, Atime, Ctime, Mode
FROM glob(globs='/**/grafana.ini',
            '/etc/grafana/*',
            '/var/lib/grafana/*',
            '/opt/grafana/*')
WHERE Mode =~ 'rw.*'  
   OR Size > 10000000

-- Check Grafana process execution patterns
SELECT Pid, Ppid, Name, CommandLine, Exe, Username, StartTime, Cwd
FROM pslist()
WHERE Name =~ 'grafana'
   OR Exe =~ 'grafana'
   OR CommandLine =~ 'grafana'

-- Hunt for recent Grafana log file modifications that may indicate unauthorized access
SELECT FullPath, Size, Mtime, Mode
FROM glob(globs='/**/grafana.log',
            '/var/log/grafana/*')
WHERE Mtime > now() - 7d  

#!/bin/bash
# Grafana Hardening and Post-Breach Verification Script
# Version: 1.0
# Purpose: Verify Grafana installation security posture post-breach notification

echo "[*] Grafana Post-Breach Security Hardening Script"
echo "[*] Starting at: $(date)"

# Check if Grafana is installed
if ! command -v grafana-server &> /dev/null
then
    echo "[!] Grafana server not found on this system."
    exit 1
fi

echo "[*] Grafana installation detected."

# Identify Grafana version
GRAFANA_VERSION=$(grafana-server -v 2>&1 | grep "Version" | awk '{print $2}')
echo "[*] Running Grafana version: $GRAFANA_VERSION"

# Check Grafana configuration file location
GRAFANA_INI="/etc/grafana/grafana.ini"
if [ ! -f "$GRAFANA_INI" ]; then
    GRAFANA_INI="/opt/grafana/conf/grafana.ini"
fi

echo "[*] Grafana configuration file: $GRAFANA_INI"

# Check for anonymous access
echo "[*] Checking for anonymous access configuration..."
if grep -q "\[auth.anonymous\]" "$GRAFANA_INI"; then
    if grep -A 5 "\[auth.anonymous\]" "$GRAFANA_INI" | grep -q "enabled = true"; then
        echo "[!] WARNING: Anonymous access is ENABLED. This is a security risk."
        echo "    Recommendation: Set enabled = false under [auth.anonymous] section."
    else
        echo "[+] Anonymous access is disabled."
    fi
else
    echo "[+] Anonymous access configuration not found (default is disabled)."
fi

# Verify authentication method configuration
echo "[*] Checking authentication method configuration..."
if grep -q "\[auth\]" "$GRAFANA_INI"; then
    AUTH_METHOD=$(grep -A 2 "\[auth\]" "$GRAFANA_INI" | grep "disable_login_form" | awk -F '=' '{print $2}' | tr -d ' ')
    if [ "$AUTH_METHOD" = "true" ]; then
        echo "[+] Login form is disabled. OAuth/SAML only."
    else
        echo "[-] Login form is enabled. Consider enforcing OAuth/SAML."
    fi
fi

# Check for exposed data sources
echo "[*] Checking for potential exposed database credentials..."
GRAFANA_DB="/var/lib/grafana/grafana.db"
if [ -f "$GRAFANA_DB" ]; then
    if command -v sqlite3 &> /dev/null; then
        echo "[*] Checking data sources for plaintext credentials..."
        sqlite3 "$GRAFANA_DB" "SELECT name FROM data_source" 2>/dev/null | while read -r dsname; do
            echo "    - Data source: $dsname"
        done
        echo "    Recommendation: Rotate all database credentials referenced in Grafana data sources."
    else
        echo "[!] sqlite3 not available to check Grafana database."
    fi
fi

# Review Grafana user accounts
echo "[*] Reviewing Grafana user accounts..."
if [ -f "$GRAFANA_DB" ] && command -v sqlite3 &> /dev/null; then
    echo "[*] Grafana users:"
    sqlite3 "$GRAFANA_DB" "SELECT login, email, role FROM user" 2>/dev/null | while read -r userline; do
        echo "    - $userline"
    done
    echo "    Recommendation: Review all admin accounts and rotate API keys."
fi

# Check for API keys
echo "[*] Checking for API keys..."
if [ -f "$GRAFANA_DB" ] && command -v sqlite3 &> /dev/null; then
    API_KEY_COUNT=$(sqlite3 "$GRAFANA_DB" "SELECT count(*) FROM api_key" 2>/dev/null)
    echo "    - Active API keys: $API_KEY_COUNT"
    echo "    Recommendation: Revoke and regenerate all API keys if any compromise is suspected."
fi

# Verify TLS/SSL configuration
echo "[*] Checking TLS configuration..."
if grep -q "\[server\]" "$GRAFANA_INI"; then
    PROTOCOL=$(grep -A 10 "\[server\]" "$GRAFANA_INI" | grep "protocol" | awk -F '=' '{print $2}' | tr -d ' ')
    CERT_FILE=$(grep -A 10 "\[server\]" "$GRAFANA_INI" | grep "cert_file" | awk -F '=' '{print $2}' | tr -d ' ')
    CERT_KEY=$(grep -A 10 "\[server\]" "$GRAFANA_INI" | grep "cert_key" | awk -F '=' '{print $2}' | tr -d ' ')
    
    if [ "$PROTOCOL" = "https" ]; then
        echo "[+] HTTPS is enabled."
        if [ -n "$CERT_FILE" ] && [ -f "$CERT_FILE" ]; then
            echo "[+] Certificate file found: $CERT_FILE"
            CERT_EXPIRY=$(openssl x509 -enddate -noout -in "$CERT_FILE" | cut -d= -f2)
            echo "    Certificate expires: $CERT_EXPIRY"
        else
            echo "[!] Certificate file not found or not configured."
        fi
    else
        echo "[!] WARNING: HTTP is in use. HTTPS is strongly recommended."
    fi
fi

# Check for potentially exposed secret files
echo "[*] Checking for potentially exposed secret files..."
SECRET_FILES=("/etc/grafana/grafana.ini" "/var/lib/grafana/grafana.db" "/opt/grafana/conf/*")
for file in "${SECRET_FILES[@]}"; do
    if [ -f "$file" ]; then
        PERMS=$(stat -c %a "$file")
        OWNER=$(stat -c %U "$file")
        echo "    File: $file"
        echo "    Permissions: $PERMS Owner: $OWNER"
        if [ "$PERMS" != "600" ] && [ "$PERMS" != "640" ]; then
            echo "    [!] WARNING: File permissions are too permissive."
        fi
    fi
done

echo ""
echo "[*] Hardening Recommendations:"
echo "1. Rotate all database credentials stored in Grafana data sources"
echo "2. Revoke and regenerate all API keys"
echo "3. Review and audit all user accounts, especially admins"
echo "4. Enable HTTPS with valid TLS certificates"
echo "5. Disable anonymous access"
echo "6. Implement OAuth/SAML for centralized identity management"
echo "7. Restrict Grafana access via network firewall rules"
echo "8. Review access logs for suspicious activity patterns"
echo "[*] Script completed at: $(date)"