CVE-2026-65110: NVIDIA Megatron-LM Checkpoint Deserialization RCE — Detection and Remediation Guide for AI/ML Infrastructure

---
title: ML Training Process Spawning Shell or Interpreter Child Process
id: 3f8a2c14-7b1d-4e59-a6c2-9d0e1f2a3b4c
status: experimental
description: Detects torchrun/python ML training processes spawning shells, scripting engines, or system utilities — consistent with malicious checkpoint deserialization payloads (CVE-2026-65110). Legitimate checkpoint loading never spawns child shells.
references:
  - http://www.zerodayinitiative.com/advisories/ZDI-26-904/
  - https://attack.mitre.org/techniques/T1059/
  - https://attack.mitre.org/techniques/T1190/
author: Security Arsenal
date: 2026/05/20
tags:
  - attack.execution
  - attack.t1059
logsource:
  category: process_creation
  product: linux
detection:
  selection_parent:
    ParentImage|endswith:
      - '/torchrun'
      - '/python'
      - '/python3'
      - '/python3.10'
      - '/python3.11'
      - '/python3.12'
  selection_parent_cli:
    ParentCommandLine|contains:
      - 'megatron'
      - 'pretrain_'
      - 'torchrun'
      - '--load'
      - 'checkpoint'
  selection_child:
    Image|endswith:
      - '/bash'
      - '/sh'
      - '/dash'
      - '/zsh'
      - '/curl'
      - '/wget'
      - '/nc'
      - '/ncat'
      - '/socat'
      - '/base64'
      - '/openssl'
      - '/perl'
      - '/ruby'
  condition: selection_parent and selection_parent_cli and selection_child
falsepositives:
  - Training wrappers that legitimately invoke curl/wget for dataset staging — tune to your data pipeline hosts
  - Slurm prolog/epilog scripts (these run under srun/slurmstepd, not python)
level: high
---
title: Reverse or Download Cradle Executed From GPU Compute Process
id: 8c1d5e62-4a3b-47f0-b9e8-2c6d0a1f3e5b
status: experimental
description: Detects common pickle-payload command patterns (curl piped to shell, base64 decode to execution, /dev/tcp reverse shells) executing on systems where ML training activity is present.
references:
  - http://www.zerodayinitiative.com/advisories/ZDI-26-904/
  - https://attack.mitre.org/techniques/T1059/004/
  - https://attack.mitre.org/techniques/T1105/
author: Security Arsenal
date: 2026/05/20
tags:
  - attack.execution
  - attack.command_and_control
  - attack.t1059.004
  - attack.t1105
logsource:
  category: process_creation
  product: linux
detection:
  selection_cradle:
    CommandLine|contains:
      - 'curl '
      - 'wget '
  selection_pipe:
    CommandLine|contains:
      - '| bash'
      - '| sh'
      - '|bash'
      - '|sh'
  selection_rev:
    CommandLine|contains:
      - '/dev/tcp/'
      - 'bash -i'
      - 'nc -e'
      - 'ncat -e'
  selection_decode:
    CommandLine|contains:
      - 'base64 -d'
      - 'base64 --decode'
  condition: (selection_cradle and selection_pipe) or selection_rev or selection_decode
falsepositives:
  - Software installation and bootstrap scripts — scope this rule to GPU compute node host groups, not developer workstations
level: high
---
title: Checkpoint File Written by Non-ML Process
id: 5e2b7f91-1c4a-43d6-8e09-6a7f2b3c4d5e
status: experimental
description: Detects checkpoint/model files (.pt, .ckpt, .distcp) being written by processes other than known training frameworks — indicating potential malicious checkpoint staging into shared model stores.
references:
  - http://www.zerodayinitiative.com/advisories/ZDI-26-904/
  - https://attack.mitre.org/techniques/T1190/
author: Security Arsenal
date: 2026/05/20
tags:
  - attack.initial_access
logsource:
  category: file_event
  product: linux
detection:
  selection_file:
    TargetFilename|endswith:
      - '.pt'
      - '.ckpt'
      - '.distcp'
      - '.safetensors'
  filter_legit:
    Image|endswith:
      - '/python'
      - '/python3'
      - '/torchrun'
      - '/rsync'
      - '/aws'
      - '/gsutil'
      - '/az'
  condition: selection_file and not filter_legit
falsepositives:
  - Administrators copying checkpoints with cp/scp — allowlist known admin service accounts and bastion hosts
  - CI/CD artifact promotion jobs — tune to known pipeline service principals
level: medium

// Hunt: Shell/interpreter children spawned from ML training processes on GPU nodes
// Ingest Linux auditd/sysmon-for-linux process events into Sentinel via Syslog/CEF,
// or hunt Windows-based ML workstations via DeviceProcessEvents.

// --- Linux (Sysmon for Linux / auditd ingested as Syslog) ---
Syslog
| where TimeGenerated > ago(7d)
| where SyslogMessage has_all ("torchrun") or SyslogMessage has_any ("pretrain_", "megatron", "--load")
| extend ParentHint = extract(@"(torchrun|python[0-9.]*)", 1, SyslogMessage)
| where SyslogMessage has_any ("/bin/bash", "/bin/sh", "/usr/bin/curl", "/usr/bin/wget", "/dev/tcp", "base64 -d")
| project TimeGenerated, Computer, ProcessName, SyslogMessage
| order by TimeGenerated desc;

// --- Windows ML workstations / MDE-instrumented nodes ---
DeviceProcessEvents
| where TimeGenerated > ago(7d)
| where InitiatingProcessFileName in~ ("python.exe", "python3.exe") or InitiatingProcessCommandLine has_any ("megatron", "torchrun", "--load", "checkpoint")
| where FileName in~ ("cmd.exe", "powershell.exe", "pwsh.exe", "wscript.exe", "mshta.exe", "curl.exe", "certutil.exe", "bitsadmin.exe")
| project TimeGenerated, DeviceName, AccountName, InitiatingProcessCommandLine, FileName, ProcessCommandLine, SHA256
| order by TimeGenerated desc;

// --- Correlation: outbound connections from GPU nodes at job-startup time ---
DeviceNetworkEvents
| where TimeGenerated > ago(7d)
| where InitiatingProcessFileName in~ ("python.exe", "python3.exe") or InitiatingProcessCommandLine has_any ("megatron", "torchrun")
| where RemoteIPType == "Public"
| where RemoteUrl !has_any ("pypi.org", "files.pythonhosted.org", "huggingface.co", "github.com", "nvidia.com", "anaconda.com")
| summarize Connections = count(), DistinctDestinations = dcount(RemoteIP) by DeviceName, InitiatingProcessCommandLine, RemoteIP, RemotePort
| order by Connections desc;

-- Artifact: SecurityArsenal.ML.CheckpointPayloadHunt
-- Purpose: Identify ML training processes with suspicious child processes or
--          unexpected network connections, consistent with malicious checkpoint
--          deserialization (CVE-2026-65110 / ZDI-26-904).

-- Pass 1: Enumerate training processes and flag shell/utility children
SELECT Pid, Ppid, Name, CommandLine, Exe, Username, CreateTime
FROM pslist()
WHERE CommandLine =~ '(megatron|torchrun|pretrain_|--load)'
   OR Name =~ '^(python|torchrun)'

-- Pass 2: Find suspicious descendants of those processes
SELECT Pid, Ppid, Name, CommandLine, Username, CreateTime
FROM pslist()
WHERE Name =~ '(bash|sh|dash|curl|wget|nc|ncat|socat|perl|ruby|openssl)'
  AND Ppid IN (
    SELECT Pid FROM pslist()
    WHERE Name =~ '^(python|torchrun)' OR CommandLine =~ '(megatron|torchrun)'
  )

-- Pass 3: Network connections held by python/torchrun processes
SELECT Pid, Name, CommandLine, LocalAddr, LocalPort, RemoteAddr, RemotePort, Status
FROM netstat()
WHERE Name =~ '(python|torchrun)'
  AND Status =~ 'ESTABLISHED'
  AND RemoteAddr !~ '^(10\\.|172\\.(1[6-9]|2[0-9]|3[01])\\.|192\\.168\\.|127\\.)'

-- Pass 4: Recently modified checkpoint files in common model directories
SELECT FullPath, Size, Mtime, Atime
FROM glob(globs=['/data/**/checkpoints/**/*.pt', '/data/**/checkpoints/**/*.distcp',
                 '/mnt/**/*.ckpt', '/home/*/**/*.pt', '/opt/**/*.ckpt'],
          accessor='file')
WHERE Mtime > (now() - 604800)  -- modified in the last 7 days
ORDER BY Mtime DESC

#!/bin/bash
# Security Arsenal — CVE-2026-65110 Megatron-LM checkpoint hygiene & hardening script
# Run on GPU compute nodes and ML workstations. Test in a non-production environment first.

set -euo pipefail
echo "=== [1] Identify Megatron-LM installations and versions ==="
# Locate Megatron-LM checkouts and installed packages
find /opt /home /usr/local -maxdepth 4 -type d -name "Megatron-LM" 2>/dev/null || true
pip list 2>/dev/null | grep -iE "megatron|torch" || true
python3 -c "import torch; print('PyTorch:', torch.__version__)" 2>/dev/null || true

echo "=== [2] Audit for unsafe torch.load usage in training code ==="
# Flag torch.load calls missing weights_only=True — the primary mitigation for pickle RCE
grep -rn --include="*.py" "torch\.load(" /opt /home 2>/dev/null \
  | grep -v "weights_only" || echo "No unsafe torch.load calls found (or no code in scanned paths)."

echo "=== [3] Scan checkpoint stores for files modified outside job windows ==="
# Adjust paths to your checkpoint stores; unexpected mtimes can indicate tampering
for dir in /data/checkpoints /mnt/models /shared/checkpoints; do
  if [ -d "$dir" ]; then
    echo "--- Recent files in $dir (last 72h):"
    find "$dir" -type f \( -name "*.pt" -o -name "*.ckpt" -o -name "*.distcp" \) -mtime -3 2>/dev/null
  fi
done

echo "=== [4] Verify checkpoint integrity tooling is in place ==="
# Recommend: hash-based allowlisting of known-good checkpoints
echo "Generating SHA256 manifest of existing checkpoints for baseline comparison..."
for dir in /data/checkpoints /mnt/models; do
  if [ -d "$dir" ]; then
    find "$dir" -type f \( -name "*.pt" -o -name "*.ckpt" \) -exec sha256sum {} \; 2>/dev/null \
      > "/var/log/checkpoint_manifest_$(date +%Y%m%d).txt"
    echo "Manifest written: /var/log/checkpoint_manifest_$(date +%Y%m%d).txt"
  fi
done

echo "=== [5] Egress hardening check for GPU nodes ==="
# GPU compute nodes should have tightly restricted egress — validate outbound rules exist
if command -v iptables &>/dev/null; then
  iptables -L OUTPUT -n --line-numbers 2>/dev/null | head -20 || echo "No OUTPUT rules visible — review egress policy."
fi

echo "=== [6] Confirm training jobs do not run as root ==="
ps aux | grep -E "torchrun|pretrain_" | grep -v grep | awk '{print $1, $11, $12}' || echo "No active training jobs."

echo "=== DONE. Update Megatron-LM per NVIDIA guidance and re-run this audit post-patch. ==="