Troubleshooting Guide¶

# Check if Sandrun is running
ps aux | grep sandrun

# Check if port is accessible
curl http://localhost:8443/

# Check system resources
free -h
df -h

# Check kernel support
cat /proc/sys/kernel/seccomp  # Should be 2
ls /proc/self/ns/               # Should list namespaces

# Check recent logs
journalctl -u sandrun -n 50

CMake Error: Could not find libseccomp

error: 'optional' is not a member of 'std'

# Check GCC version (need 7.0+)
gcc --version

# Update if needed (Ubuntu)
sudo apt-get install g++-9
export CXX=g++-9

# Clean and rebuild
rm -rf build
cmake -B build -DCMAKE_CXX_COMPILER=g++-9
cmake --build build

CMake 3.10 or higher is required. You are running version 2.8

# Install latest CMake
wget https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-linux-x86_64.sh
chmod +x cmake-3.25.0-linux-x86_64.sh
sudo ./cmake-3.25.0-linux-x86_64.sh --prefix=/usr/local --skip-license

# Verify
cmake --version

[ERROR] Failed to create namespace: Operation not permitted

[ERROR] Failed to bind to port 8443: Address already in use

# Option 1: Find what's using the port
sudo lsof -i :8443
sudo netstat -tulpn | grep 8443

# Option 2: Kill the process
sudo kill <PID>

# Option 3: Use different port
sudo ./build/sandrun --port 9000

# Option 4: Stop existing Sandrun instance
sudo systemctl stop sandrun
# or
sudo pkill -f sandrun

[ERROR] Seccomp not supported on this kernel

# Check seccomp support
cat /proc/sys/kernel/seccomp
# Should output: 2

# If 0, rebuild kernel with CONFIG_SECCOMP=y
# Or use a distribution with seccomp enabled

# Check kernel version (need 4.6+)
uname -r

# Update kernel if needed
sudo apt-get install linux-generic-hwe-20.04
sudo reboot

[ERROR] Failed to open worker key: /etc/sandrun/worker.pem: No such file or directory

# Generate new worker key
sudo mkdir -p /etc/sandrun
sudo ./build/sandrun --generate-key /etc/sandrun/worker.pem

# Or skip worker key if not using pools
sudo ./build/sandrun --port 8443  # No --worker-key flag

{
  "error": "Invalid manifest",
  "details": "Missing required field: entrypoint"
}

# Validate JSON syntax
echo '{"entrypoint":"main.py"}' | jq .

# Minimum valid manifest
cat > manifest.json <<EOF
{
  "entrypoint": "main.py",
  "interpreter": "python3"
}
EOF

# Submit with manifest
curl -X POST http://localhost:8443/submit \
  -F "files=@job.tar.gz" \
  -F "manifest=$(cat manifest.json)"

{
  "error": "Upload too large",
  "details": "Maximum size: 100MB"
}

# Check tarball size
ls -lh job.tar.gz

# Compress better
tar czf job.tar.gz --best my_project/

# Remove unnecessary files
tar czf job.tar.gz \
  --exclude='*.pyc' \
  --exclude='__pycache__' \
  --exclude='.git' \
  --exclude='node_modules' \
  my_project/

# Split into multiple jobs if needed

# List tarball contents
tar -tzf job.tar.gz

# Ensure entrypoint is included
tar -tzf job.tar.gz | grep main.py

# Create tarball correctly
cd project_directory
tar czf ../job.tar.gz .
# Not: tar czf job.tar.gz project_directory

{
  "error": "Rate limit exceeded",
  "reason": "CPU quota exhausted (10.2/10.0 seconds used)",
  "retry_after": 45
}

# Check your quota
curl http://localhost:8443/stats

# Wait for quota to reset
sleep 60

# Optimize code to use less CPU
# Split long jobs into smaller chunks
# Use more efficient algorithms

{
  "status": "failed",
  "exit_code": 1
}

# Get full logs
curl http://localhost:8443/logs/job-abc123

# Common causes:
# - Syntax error in code
# - Missing dependency
# - File not found
# - Permission error

# Test locally first
cd project_directory
python3 main.py  # Test before submitting

# Add debugging
cat > main.py <<EOF
import sys
print("Python version:", sys.version)
print("Working directory:", os.getcwd())
print("Files:", os.listdir('.'))
# ... your code ...
EOF

{
  "status": "failed",
  "exit_code": 137,
  "details": "Killed by signal 9"
}

# Increase memory limit in manifest
cat > manifest.json <<EOF
{
  "entrypoint": "main.py",
  "memory_mb": 1024
}
EOF

# Increase timeout
cat > manifest.json <<EOF
{
  "entrypoint": "main.py",
  "timeout": 600
}
EOF

# Optimize memory usage
# - Use generators instead of lists
# - Process data in chunks
# - Delete large objects when done

PermissionError: [Errno 13] Permission denied: '/etc/passwd'

# Copy needed files into job directory
cp /etc/hosts my_project/hosts
tar czf job.tar.gz my_project/

# In your code, use relative paths
with open('hosts', 'r') as f:  # Not /etc/hosts
    data = f.read()

ModuleNotFoundError: No module named 'numpy'

# Check system stats
curl http://localhost:8443/stats

# Response shows:
# "queue_length": 10  # Many queued jobs
# "active_jobs": 2    # System busy

# Wait for active jobs to complete
# Or cancel queued jobs if needed

# Check worker health (pool deployments)
curl http://pool:9000/pool

{
  "error": "Job not found or expired"
}

# Download immediately after completion
# Check status first
STATUS=$(curl -s http://localhost:8443/status/job-abc123 | jq -r '.status')

if [ "$STATUS" = "completed" ]; then
  curl http://localhost:8443/download/job-abc123/output.txt -o output.txt
fi

# Use WebSocket streaming to monitor completion

WARNING:__main__:No available workers for job pool-xxx

# Check pool status
curl http://pool:9000/pool

# Response shows:
# "healthy_workers": 0

# Check worker health directly
curl http://worker1:8443/health

# Start more workers
sudo ./build/sandrun --port 8443 --worker-key /etc/sandrun/worker.pem

# Check worker logs
journalctl -u sandrun -f

# Verify workers.json configuration
cat workers.json
# Ensure worker IDs and endpoints are correct

ERROR:__main__:Health check failed for worker: Invalid worker_id

# Regenerate worker key
sudo ./build/sandrun --generate-key /etc/sandrun/worker.pem

# Copy output Worker ID
# Update workers.json with new worker_id

# Restart worker
sudo systemctl restart sandrun

# Check pool logs
journalctl -u pool-coordinator -f

# Check worker endpoints
for worker in worker1 worker2 worker3; do
  echo "Testing $worker:"
  curl http://$worker:8443/health
done

# Verify network connectivity
ping worker1
telnet worker1 8443

# Check firewall rules
sudo iptables -L -n

# Ensure workers are reachable from coordinator
# Update workers.json with correct endpoints

# Check system load
uptime
top

# Check I/O wait
iostat -x 1

# Check memory pressure
free -h
vmstat 1

# Increase system resources
# Add more RAM
# Use faster CPU
# Add more workers for horizontal scaling

# Optimize job code
# Use compiled languages for CPU-intensive tasks
# Minimize disk I/O
# Use efficient algorithms

# Check memory usage
free -h

# Check tmpfs usage
df -h /dev/shm
mount | grep tmpfs

# Check per-process memory
ps aux --sort=-%mem | head -20

# Increase system RAM
# Reduce concurrent job limit
# Reduce per-job memory limit
# Optimize job code for memory efficiency

# Clean up old jobs manually if needed
sudo systemctl restart sandrun

# Build with debug symbols
cmake -B build -DCMAKE_BUILD_TYPE=Debug
cmake --build build

# Run with verbose logging
sudo ./build/sandrun --port 8443 --verbose

# Or set environment variable
export SANDRUN_LOG_LEVEL=debug
sudo -E ./build/sandrun --port 8443

# Build with debug symbols
cmake -B build -DCMAKE_BUILD_TYPE=Debug
cmake --build build

# Run under GDB
sudo gdb ./build/sandrun
(gdb) run --port 8443
# Wait for crash
(gdb) backtrace
(gdb) info locals

# Run with Valgrind
sudo valgrind \
  --leak-check=full \
  --show-leak-kinds=all \
  --track-origins=yes \
  --verbose \
  --log-file=valgrind.log \
  ./build/sandrun --port 8443

# Analyze results
cat valgrind.log

# Monitor HTTP traffic
sudo tcpdump -i any -nn -A port 8443

# Or use Wireshark
sudo wireshark

# Test with verbose curl
curl -v http://localhost:8443/