Massed Compute

Connect and Verify Prerequisites

SSH to your L40 host and confirm the system is ready for diagnostic collection.

ssh YOUR_VM_IP

# Verify system readiness
hostname; ip -br addr show scope global
df -h ~ | awk 'NR==2 {print $4 " free"}'
sudo -n true 2>/dev/null && echo "sudo ok" || echo "sudo will prompt"
python3 -c 'import re, os, ipaddress' && echo "python3 ok"
which nvidia-smi >/dev/null && nvidia-smi -L | head -5 || echo "WARN: nvidia-smi not found"

Ensure you have at least 200 MB free space and that nvidia-smi returns GPU information.

Set Optional Application Log Path

If you want to include application-specific logs in the bundle, set the APP_LOG environment variable.

# Optional: include the last 200 lines of your workload log
export APP_LOG=/path/to/your/application.log

The collector will include the tail of this log file if it exists and is readable.

Run the Debug Collection Script

Execute the complete collection and redaction script. This creates a timestamped directory and gathers all diagnostic data.

# Complete debug bundle collection script
set -euo pipefail
HOST=$(hostname -s)
TS=$(date +%Y%m%dT%H%M%SZ)
BUNDLE=~/debug-bundles/debug-bundle-${HOST}-${TS}
mkdir -p "$BUNDLE"/{identity,hardware,gpu,driver,cuda,logs,network,disk,workload,app}

# 1. System identity
hostname > "$BUNDLE/identity/hostname.txt"
uname -a > "$BUNDLE/identity/uname.txt"
cat /etc/os-release > "$BUNDLE/identity/os-release.txt"
uptime > "$BUNDLE/identity/uptime.txt"
who > "$BUNDLE/identity/who.txt" || true
last -F 2>/dev/null | head -10 > "$BUNDLE/identity/last.txt" || true

# 2. Hardware information
lspci -nn 2>/dev/null | grep -iE 'nvidia|vga|3d controller' > "$BUNDLE/hardware/lspci-summary.txt" || true
PCI_IDS=$(lspci -nn 2>/dev/null | awk '/NVIDIA/{print $1}')
for id in $PCI_IDS; do
  sudo lspci -v -s "$id" > "$BUNDLE/hardware/lspci-v-${id//[:.]/_}.txt" 2>&1 || true
done
sudo dmidecode -t system 2>/dev/null > "$BUNDLE/hardware/dmidecode-system.txt" || true

# 3. GPU diagnostics
nvidia-smi -q > "$BUNDLE/gpu/nvidia-smi-q.txt" 2>&1 || echo "nvidia-smi -q failed" > "$BUNDLE/gpu/nvidia-smi-q.txt"
nvidia-smi --query-gpu=index,name,driver_version,vbios_version,memory.total,memory.used,utilization.gpu,utilization.memory,temperature.gpu,power.draw,ecc.errors.uncorrected.volatile.total,ecc.errors.uncorrected.aggregate.total --format=csv > "$BUNDLE/gpu/nvidia-smi-csv.txt" 2>&1 || true
nvidia-smi topo -m > "$BUNDLE/gpu/nvidia-smi-topo.txt" 2>&1 || true
nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv > "$BUNDLE/gpu/compute-apps.csv" 2>&1 || true

Continue Data Collection

The script continues collecting driver, CUDA, system logs, and network information.

# 4. Driver and kernel info
dpkg -l 2>/dev/null | grep -iE 'nvidia|cuda' > "$BUNDLE/driver/dpkg-nvidia.txt" || true
lsmod | grep -E '^nvidia' > "$BUNDLE/driver/lsmod-nvidia.txt" || true
modinfo nvidia 2>/dev/null | head -20 > "$BUNDLE/driver/modinfo-nvidia.txt" || true
uname -r > "$BUNDLE/driver/kernel.txt"
dpkg -l 2>/dev/null | grep -E "linux-(image|headers)-$(uname -r)" > "$BUNDLE/driver/dpkg-kernel.txt" || true

# 5. CUDA installation
which nvcc > "$BUNDLE/cuda/which-nvcc.txt" 2>&1 || true
nvcc --version > "$BUNDLE/cuda/nvcc-version.txt" 2>&1 || true
ls -ld /usr/local/cuda* 2>/dev/null > "$BUNDLE/cuda/usr-local-cuda.txt" || true
ldconfig -p 2>/dev/null | grep -iE 'cuda|cudnn|nccl' > "$BUNDLE/cuda/ldconfig-cuda.txt" || true

# 6. System logs
sudo dmesg -T 2>/dev/null | tail -500 > "$BUNDLE/logs/dmesg-tail.txt" || true
sudo journalctl -k -b 0 2>/dev/null | tail -500 > "$BUNDLE/logs/journal-kernel-tail.txt" || true
sudo journalctl -u nvidia-persistenced --since "1 hour ago" 2>/dev/null > "$BUNDLE/logs/journal-nvidia-persistenced.txt" || true
sudo journalctl -p err -b 0 2>/dev/null | tail -200 > "$BUNDLE/logs/journal-errors.txt" || true

Complete Collection and Redaction

Finish data collection, run the Python redactor, and create the final tarball.

# 7. Network configuration
ip -br addr > "$BUNDLE/network/ip-addr.txt"
ip route > "$BUNDLE/network/ip-route.txt"
cat /etc/resolv.conf > "$BUNDLE/network/resolv-conf.txt" 2>/dev/null || true
ss -ltnp > "$BUNDLE/network/ss-listen.txt" 2>/dev/null || true
sudo iptables -L -n 2>/dev/null | head -50 > "$BUNDLE/network/iptables.txt" || true

# 8. Disk usage
df -h > "$BUNDLE/disk/df-h.txt"
df -i > "$BUNDLE/disk/df-i.txt"
du -sh ~/* 2>/dev/null | sort -h | tail > "$BUNDLE/disk/du-home.txt" || true

# 9. Running processes and environment
ps -ef --sort=-%mem 2>/dev/null | head -20 > "$BUNDLE/workload/ps-mem.txt" || true
env > "$BUNDLE/workload/env.txt"

# Include application log if specified
APP_LOG="${APP_LOG:-}"
if [ -n "$APP_LOG" ] && [ -r "$APP_LOG" ]; then
  tail -200 "$APP_LOG" > "$BUNDLE/app/$(basename "$APP_LOG").tail.txt"
fi

Apply Privacy Redaction

Run the automated redaction to remove sensitive information from all collected files.

# Redaction using Python
RED_COUNTS=$(python3 - "$BUNDLE" <<'PY'
import os, re, sys, ipaddress
root = sys.argv[1]
ips = tokens = macs = paths = 0
ip_re   = re.compile(r'(?'; ips += 1
    return ip_map[s]
def repl_mac(m):
    global macs; macs += 1; return ''
def repl_tok(m):
    global tokens; tokens += 1; return ''
def repl_home(m):
    global paths; paths += 1; return ''
def repl_env(m):
    global tokens; tokens += 1; return m.group(1) + '='
for dp, _, fs in os.walk(root):
    for f in fs:
        p = os.path.join(dp, f)
        try:
            with open(p, 'r', errors='replace') as fh: s = fh.read()
        except Exception:
            continue
        s = ip_re.sub(repl_ip, s)
        s = mac_re.sub(repl_mac, s)
        s = tok_re.sub(repl_tok, s)
        s = home_re.sub(repl_home, s)
        s = env_re.sub(repl_env, s)
        with open(p, 'w') as fh: fh.write(s)
print(f"{ips} {tokens} {macs} {paths}")
PY
)

Create Final Bundle

Package everything into a compressed tarball with manifest and verification information.

# Parse redaction results and create summary
read R_IPS R_TOKENS R_MACS R_PATHS <<< "$RED_COUNTS"
echo "Redacted: ${R_IPS} IPs, ${R_TOKENS} tokens, ${R_MACS} MACs, ${R_PATHS} home paths" \
  | tee "$BUNDLE/REDACTION_SUMMARY.txt"

# Create manifest and tarball
( cd "$BUNDLE" && find . -type f -printf '%s\t%p\n' | sort -n ) > "$BUNDLE/MANIFEST.txt"
TARBALL="${BUNDLE}.tar.gz"
tar -czf "$TARBALL" -C "$(dirname "$BUNDLE")" "$(basename "$BUNDLE")"
chmod -R go-rwx "$BUNDLE" "$TARBALL"

# Generate verification info
SHA=$(sha256sum "$TARBALL" | awk '{print $1}')
BYTES=$(stat -c%s "$TARBALL")
COUNT=$(tar -tzf "$TARBALL" | wc -l)

echo "---"
echo "bundle: $TARBALL"
echo "sha256: $SHA"
echo "bytes:  $BYTES"
echo "files:  $COUNT"
echo "redaction: $(cat "$BUNDLE/REDACTION_SUMMARY.txt")"

Download Bundle

Transfer the completed debug bundle to your local machine for analysis or sharing.

# From your local machine
mkdir -p ./debug-bundles
scp YOUR_VM_IP:/home/ubuntu/debug-bundles/debug-bundle-*.tar.gz ./debug-bundles/

# Verify bundle contents
tar -tzf ./debug-bundles/debug-bundle-*.tar.gz | head -20
tar -xOf ./debug-bundles/debug-bundle-*.tar.gz '*/REDACTION_SUMMARY.txt'