From c5d7b777e11e4607ba4f8d6a74c4cb704909ff4a Mon Sep 17 00:00:00 2001 From: Csaba Kecskemeti Date: Fri, 23 Jan 2026 18:39:02 -0800 Subject: [PATCH 1/6] initial version of the playbook --- .../DISTRIBUTED-INFERENCE.md | 528 ++++++++++++++ .../README.md | 647 ++++++++++++++++++ .../assets/devquasar-logo.png | Bin 0 -> 36156 bytes .../assets/test_nccl.py | 96 +++ 4 files changed, 1271 insertions(+) create mode 100644 nvidia/heterogeneous-distributed-inference-rdma/DISTRIBUTED-INFERENCE.md create mode 100644 nvidia/heterogeneous-distributed-inference-rdma/README.md create mode 100644 nvidia/heterogeneous-distributed-inference-rdma/assets/devquasar-logo.png create mode 100644 nvidia/heterogeneous-distributed-inference-rdma/assets/test_nccl.py diff --git a/nvidia/heterogeneous-distributed-inference-rdma/DISTRIBUTED-INFERENCE.md b/nvidia/heterogeneous-distributed-inference-rdma/DISTRIBUTED-INFERENCE.md new file mode 100644 index 0000000..4a68729 --- /dev/null +++ b/nvidia/heterogeneous-distributed-inference-rdma/DISTRIBUTED-INFERENCE.md @@ -0,0 +1,528 @@ +# Distributed Inference Guide + +> Deploy and run distributed AI inference across DGX Spark and Linux Workstation using vLLM and Ray + +## Table of Contents + +- [Overview](#overview) +- [Instructions](#instructions) +- [Performance Benchmarks](#performance-benchmarks) +- [Troubleshooting](#troubleshooting) +- [Credits](#credits) + +--- + +## Overview + +## Basic idea + +This guide walks you through deploying distributed inference across your heterogeneous RDMA cluster. Using Ray for orchestration and vLLM for inference, you can run large language models that exceed the memory capacity of any single GPU by distributing them across your DGX Spark and Linux workstation. + +**Architecture:** +``` +┌─────────────────────────────────┐ ┌───────────────────────────────────┐ +│ DGX SPARK │ │ WORKSTATION │ +│ (Grace Blackwell GB10) │ │ (RTX 6000 Pro / RTX 5090) │ +│ │ │ │ +│ ┌───────────────────────────┐ │ │ ┌───────────────────────────┐ │ +│ │ vLLM Head Node │ │ │ │ vLLM Worker │ │ +│ │ (API Server, Rank 0) │ │ │ │ (Tensor Parallel) │ │ +│ └───────────────────────────┘ │ │ └───────────────────────────┘ │ +│ │ │ │ │ │ +│ ┌───────────────────────────┐ │ │ ┌───────────────────────────┐ │ +│ │ Ray Head (6379) │◄─┼────┼──│ Ray Worker │ │ +│ └───────────────────────────┘ │ │ └───────────────────────────┘ │ +│ │ │ │ │ │ +│ ┌───────────────────────────┐ │ │ ┌───────────────────────────┐ │ +│ │ NCCL over RDMA │◄─┼════┼──│ NCCL over RDMA │ │ +│ │ 192.168.200.1 │ │ │ │ 192.168.200.2 │ │ +│ └───────────────────────────┘ │ │ └───────────────────────────┘ │ +└─────────────────────────────────┘ └───────────────────────────────────┘ +``` + +## What you'll accomplish + +- Configure SSH and hostname resolution between nodes +- Test NCCL communication over RDMA +- Deploy RDMA-enabled Docker containers +- Establish a Ray cluster across both systems +- Run distributed inference with vLLM +- Benchmark performance across different configurations + +## What to know before starting + +- Familiarity with Docker and container networking +- Understanding of distributed computing concepts (Ray, tensor parallelism) +- Basic knowledge of LLM inference serving + +## Prerequisites + +- Completed [RDMA Network Setup](README.md) with validated 90+ Gbps bandwidth +- Docker installed on both systems: `docker --version` +- NVIDIA Container Toolkit installed +- Hugging Face account for model access (some models require authentication) + +## Time & risk + +- **Duration:** 1-2 hours including testing + +- **Risk level:** Low - uses containers, non-destructive + +- **Rollback:** Stop containers to revert + +- **Last Updated:** 01/23/2026 + +--- + +## Instructions + +## Step 1. Configure Hostnames + +Add hostname aliases on both systems: + +```bash +## Add hostname resolution on both DGX Spark and Workstation +sudo tee -a /etc/hosts > /dev/null <@workstation +``` + +On Workstation: +```bash +## Check if SSH key exists +ls ~/.ssh/id_*.pub + +## If no key exists, generate one: +ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 + +## Copy key to DGX Spark +ssh-copy-id @dgx-spark +``` + +Verify passwordless SSH: + +```bash +## From DGX Spark +ssh @workstation hostname +## Expected output: workstation + +## From Workstation +ssh @dgx-spark hostname +## Expected output: dgx-spark +``` + +--- + +## Step 3. Test NCCL Communication + +Create the NCCL test script on both systems: + +```bash +## Create test script +cat > test_nccl.py << 'EOF' +import os +import torch +import torch.distributed as dist +import argparse + +def test_nccl_communication(): + parser = argparse.ArgumentParser() + parser.add_argument('--rank', type=int, required=True) + parser.add_argument('--world_size', type=int, default=2) + parser.add_argument('--master_addr', type=str, default='192.168.200.1') + parser.add_argument('--master_port', type=str, default='29500') + args = parser.parse_args() + + os.environ['RANK'] = str(args.rank) + os.environ['WORLD_SIZE'] = str(args.world_size) + os.environ['MASTER_ADDR'] = args.master_addr + os.environ['MASTER_PORT'] = args.master_port + os.environ['NCCL_SOCKET_IFNAME'] = 'enp1s0f0np0' + + print(f"Initializing process group - Rank: {args.rank}, World Size: {args.world_size}") + print(f"Master: {args.master_addr}:{args.master_port}") + + dist.init_process_group(backend='nccl', rank=args.rank, world_size=args.world_size) + print(f"Process group initialized - Rank: {dist.get_rank()}/{dist.get_world_size()}") + + device = torch.device('cuda:0') + tensor = torch.ones(10, device=device) * (args.rank + 1) + print(f"Rank {args.rank} - Before allreduce: {tensor}") + + dist.all_reduce(tensor, op=dist.ReduceOp.SUM) + print(f"Rank {args.rank} - After allreduce: {tensor}") + print(f"Expected result: {torch.ones(10) * (1 + 2)}") + + dist.destroy_process_group() + print(f"Rank {args.rank} - Test completed successfully!") + +if __name__ == "__main__": + test_nccl_communication() +EOF +``` + +Run NCCL test in Docker containers: + +On DGX Spark (start first): +```bash +docker run -it --runtime=nvidia --gpus all --network host --ipc=host \ + --privileged --ulimit memlock=-1 --ulimit stack=67108864 \ + -v /dev/infiniband:/dev/infiniband -v /sys:/sys:ro \ + -e NCCL_IB_DISABLE=0 -e NCCL_IB_HCA=rocep1s0f0:1 -e NCCL_IB_GID_INDEX=3 \ + -e NCCL_SOCKET_IFNAME=enp1s0f0np0 -v $(pwd):/workspace \ + nvcr.io/nvidia/vllm:25.09-py3 python /workspace/test_nccl.py --rank 0 +``` + +On Workstation (connect to DGX): +```bash +docker run -it --runtime=nvidia --gpus all --network host --ipc=host \ + --privileged --ulimit memlock=-1 --ulimit stack=67108864 \ + -v /dev/infiniband:/dev/infiniband -v /sys:/sys:ro \ + -e NCCL_IB_DISABLE=0 -e NCCL_IB_HCA=rocep1s0f0:1 -e NCCL_IB_GID_INDEX=3 \ + -e NCCL_SOCKET_IFNAME=enp1s0f0np0 -v $(pwd):/workspace \ + nvcr.io/nvidia/vllm:25.09-py3 python /workspace/test_nccl.py --rank 1 +``` + +**Success indicators:** +- Output shows: `NCCL INFO Using network IBext_v10` +- All-reduce operation completes successfully +- Final tensors show expected sum values (3.0 for each element) + +--- + +## Step 4. Start RDMA-Enabled Containers + +On DGX Spark: +```bash +docker run -it --runtime=nvidia --gpus all --network host --ipc=host --shm-size=10g \ + --privileged \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + -v /dev/infiniband:/dev/infiniband \ + -v /sys:/sys:ro \ + -e CUDA_DEVICE_ORDER=PCI_BUS_ID \ + -e GLOO_SOCKET_IFNAME=enp1s0f0np0 \ + -e NCCL_IB_DISABLE=0 \ + -e NCCL_IB_HCA=rocep1s0f0:1 \ + -e NCCL_IB_GID_INDEX=3 \ + -e NCCL_SOCKET_IFNAME=enp1s0f0np0 \ + -e RAY_USE_MULTIPLE_IPS=0 \ + -e RAY_NODE_IP_ADDRESS=192.168.200.1 \ + -e RAY_OVERRIDE_NODE_IP=192.168.200.1 \ + -e VLLM_HOST_IP=192.168.200.1 \ + nvcr.io/nvidia/vllm:25.09-py3 bash +``` + +On Workstation: +```bash +docker run -it --runtime=nvidia --gpus all --network host --ipc=host --shm-size=10g \ + --privileged \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + -v /dev/infiniband:/dev/infiniband \ + -v /sys:/sys:ro \ + -e CUDA_DEVICE_ORDER=PCI_BUS_ID \ + -e GLOO_SOCKET_IFNAME=enp1s0f0np0 \ + -e NCCL_IB_DISABLE=0 \ + -e NCCL_IB_HCA=rocep1s0f0:1 \ + -e NCCL_IB_GID_INDEX=3 \ + -e NCCL_SOCKET_IFNAME=enp1s0f0np0 \ + -e RAY_USE_MULTIPLE_IPS=0 \ + -e RAY_NODE_IP_ADDRESS=192.168.200.2 \ + -e RAY_OVERRIDE_NODE_IP=192.168.200.2 \ + nvcr.io/nvidia/vllm:25.09-py3 bash +``` + +**Key parameters explained:** +- `--runtime=nvidia`: Required for GPU access +- `--network host`: Uses host networking (required for RDMA) +- `--privileged`: Needed for InfiniBand device access +- `--ulimit memlock=-1`: Unlimited memory locking for RDMA +- `-v /dev/infiniband:/dev/infiniband`: Mounts RDMA devices +- `NCCL_IB_HCA=rocep1s0f0:1`: Tells NCCL to use specific RDMA device +- `RAY_USE_MULTIPLE_IPS=0`: Prevents Ray IP detection issues + +--- + +## Step 5. Establish Ray Cluster + +On DGX Spark container (head node): +```bash +ray start --head \ + --node-ip-address=192.168.200.1 \ + --port=6379 \ + --dashboard-host=192.168.200.1 \ + --dashboard-port=8265 \ + --num-gpus=1 +``` + +Verify head node: +```bash +ray status +``` + +Expected output: +``` +======== Autoscaler status: 2026-01-10 19:43:05.517578 ======== +Node status +--------------------------------------------------------------- +Active: + 1 node_xxxxx +Resources +--------------------------------------------------------------- +Total Usage: + 0.0/20.0 CPU + 0.0/1.0 GPU +``` + +On Workstation container (worker node): +```bash +ray start \ + --address=192.168.200.1:6379 \ + --node-ip-address=192.168.200.2 \ + --num-gpus=2 +``` + +Verify cluster formation: +```bash +ray status +``` + +Expected output (should show 3 total GPUs): +``` +======== Autoscaler status: 2026-01-10 19:46:26.274139 ======== +Node status +--------------------------------------------------------------- +Active: + 1 node_xxxxx (head) + 1 node_xxxxx (worker) +Resources +--------------------------------------------------------------- +Total Usage: + 0.0/68.0 CPU + 0.0/3.0 GPU +``` + +--- + +## Step 6. Run Validation Test (4B Model) + +Start small model for validation on DGX Spark container: + +```bash +python -m vllm.entrypoints.openai.api_server \ + --model Qwen/Qwen3-4B-Instruct-2507 \ + --tensor-parallel-size 2 \ + --distributed-executor-backend ray \ + --gpu-memory-utilization 0.8 \ + --host 192.168.200.1 \ + --port 8000 +``` + +Test from another terminal: +```bash +curl -X POST "http://192.168.200.1:8000/v1/completions" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen/Qwen3-4B-Instruct-2507", + "prompt": "Test distributed inference:", + "max_tokens": 500 + }' +``` + +--- + +## Step 7. Run FP8 Quantized Model (30B) + +FP8 quantization provides excellent memory efficiency with good performance: + +```bash +## Stop previous model (Ctrl+C), then start FP8 30B model +python -m vllm.entrypoints.openai.api_server \ + --model Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 \ + --tensor-parallel-size 2 \ + --distributed-executor-backend ray \ + --gpu-memory-utilization 0.8 \ + --host 192.168.200.1 \ + --port 8000 +``` + +**Benefits of FP8:** +- Memory efficiency: Reduced footprint compared to BF16 +- Performance: 341+ tok/s demonstrated +- Hardware compatibility: Fully supported on Blackwell GB10 + +--- + +## Step 8. Run Production Model (72B) + +Memory-optimized configuration for 136GB model: + +```bash +python -m vllm.entrypoints.openai.api_server \ + --model Qwen/Qwen2.5-72B-Instruct \ + --tensor-parallel-size 2 \ + --distributed-executor-backend ray \ + --gpu-memory-utilization 0.85 \ + --host 192.168.200.1 \ + --port 8000 \ + --max-model-len 2048 \ + --max-num-seqs 8 \ + --disable-sliding-window \ + --enforce-eager +``` + +**Memory optimization parameters:** +- `--gpu-memory-utilization 0.85`: Uses 85% of GPU memory +- `--max-model-len 2048`: Limits context length to save memory +- `--max-num-seqs 8`: Reduces concurrent sequences +- `--disable-sliding-window`: Disables memory-intensive sliding window attention +- `--enforce-eager`: Uses eager execution (saves memory) + +Test 72B model: +```bash +curl -X POST "http://192.168.200.1:8000/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen/Qwen2.5-72B-Instruct", + "messages": [ + {"role": "user", "content": "Explain the benefits of RDMA for AI workloads in one paragraph."} + ], + "max_tokens": 500 + }' +``` + +--- + +## Step 9. Monitor RDMA Traffic + +Monitor RDMA activity during inference: + +```bash +## Run on both systems (separate terminals) +watch -n 0.5 " +echo '=== RDMA Counters ==='; +echo -n 'TX: '; cat /sys/class/infiniband/rocep1s0f0/ports/1/counters/port_xmit_data; +echo -n 'RX: '; cat /sys/class/infiniband/rocep1s0f0/ports/1/counters/port_rcv_data; +echo 'Timestamp: '; date; +" +``` + +During inference, you'll see counters increasing as tensors are communicated between GPUs. + +--- + +## Performance Benchmarks + +### Benchmark Commands + +**Single-node testing:** +```bash +## On RTX 6000 Pro or DGX Spark +vllm bench latency --model Qwen/Qwen3-30B-A3B-Thinking-2507 --input-len 512 --output-len 2000 --num-iters 10 +vllm bench throughput --model Qwen/Qwen3-30B-A3B-Thinking-2507 --input-len 512 --output-len 2000 --num-prompts 20 +``` + +**Distributed testing:** +```bash +## 30B Model +vllm bench serve --host 192.168.200.1 --port 8000 --random-input-len 512 --random-output-len 2000 --num-prompts 20 --request-rate 2 --model Qwen/Qwen3-30B-A3B-Thinking-2507 + +## 72B Model +vllm bench serve --host 192.168.200.1 --port 8000 --random-input-len 256 --random-output-len 1500 --num-prompts 20 --request-rate 2 --model Qwen/Qwen2.5-72B-Instruct +``` + +### Performance Results Summary + +| Configuration | Avg Latency | Output Throughput | Total Throughput | +|---------------|-------------|-------------------|------------------| +| **RTX 6000 Pro (Single)** | 36.87s | 679.88 tok/s | 853.90 tok/s | +| **DGX Spark (Single)** | 213.12s | 105.10 tok/s | 132.00 tok/s | +| **Distributed RDMA** | 191.09s | 205.83 tok/s | 259.41 tok/s | + +### Key Insights + +**RTX 6000 Pro: Clear Single-Node Winner** +- 5.8x faster than DGX Spark for latency-critical workloads +- 6.5x higher output token throughput +- Best for: Interactive inference, real-time applications + +**Distributed RDMA: Aggregated Capacity** +- 259.41 tok/s total throughput - faster than DGX alone +- Combined 224GB GPU memory (128GB DGX + 96GB RTX) +- Enables models too large for any single GPU +- TTFT: 139.94ms mean vs single DGX 213,120ms + +**DGX Spark: Memory Advantage** +- 128GB unified memory enables larger models +- Slower inference but handles 100B+ models +- Best for: Extremely large models, memory-constrained scenarios + +### FP8 30B Model Results + +``` +============ Serving Benchmark Result ============ +Successful requests: 20 +Benchmark duration (s): 115.36 +Output token throughput (tok/s): 341.15 +Total Token throughput (tok/s): 429.89 +Mean TTFT (ms): 171.00 +Mean TPOT (ms): 53.08 +================================================== +``` + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---------|-------|-----| +| Ray worker can't connect to head | Firewall blocking port 6379 | `sudo ufw allow 6379/tcp` | +| NCCL timeout during model load | RDMA not working | Verify `ib_send_bw` test passes | +| "Placement group" errors | Ray cluster not formed | Check `ray status` on both nodes | +| OOM during 72B model load | Insufficient memory optimization | Add `--max-model-len 2048 --enforce-eager` | +| SSH connection refused | SSH server not running | `sudo systemctl start ssh` | +| Container can't access RDMA | Missing device mount | Ensure `-v /dev/infiniband:/dev/infiniband` | +| Wrong IP in Ray cluster | Multiple network interfaces | Set `RAY_USE_MULTIPLE_IPS=0` | +| Slow inference performance | NCCL using wrong interface | Verify `NCCL_SOCKET_IFNAME=enp1s0f0np0` | + +--- + +## Credits + +This playbook was contributed by **Csaba Kecskemeti** | [DevQuasar](https://devquasar.com/). + +For a detailed walkthrough and additional context, see the original article: +[Distributed Inference Cluster: DGX Spark + RTX 6000 Pro](https://devquasar.com/ai/edge-ai/distributed-inference-cluster-dgx-spark-rtx-6000-pro/) + +![DevQuasar](assets/devquasar-logo.png) diff --git a/nvidia/heterogeneous-distributed-inference-rdma/README.md b/nvidia/heterogeneous-distributed-inference-rdma/README.md new file mode 100644 index 0000000..05d544e --- /dev/null +++ b/nvidia/heterogeneous-distributed-inference-rdma/README.md @@ -0,0 +1,647 @@ +# Heterogeneous Distributed Inference over RDMA + +> Set up high-speed RDMA networking between DGX Spark (ConnectX-7) and a Linux Workstation (ConnectX-5) for distributed AI inference + +## Table of Contents + +- [Overview](#overview) +- [Instructions](#instructions) +- [Troubleshooting](#troubleshooting) +- [Next Steps](#next-steps) +- [Credits](#credits) + +--- + +## Overview + +## Basic idea + +This playbook guides you through setting up a heterogeneous distributed computing environment using RDMA (Remote Direct Memory Access) over Converged Ethernet (RoCE v2). You will connect a DGX Spark system with a Linux workstation equipped with a Mellanox ConnectX network adapter, enabling high-speed GPU-to-GPU communication for distributed AI workloads. + +With RDMA enabled, data flows directly between GPU memories: + +``` +GPU memory → PCIe → NIC (mlx5) → wire → NIC → PCIe → GPU memory +``` + +**Key properties:** +- **No CPU copies:** Data bypasses system memory +- **No kernel networking stack:** Direct hardware-to-hardware communication +- **Ultra-low latency:** ~750 nanoseconds end-to-end +- **High message rate:** Up to 200M messages/second + +## What you'll accomplish + +- Enable low-latency, zero-copy GPU↔GPU communication between heterogeneous systems +- Configure RoCE v2 networking over 100 Gbps direct QSFP connection +- Validate RDMA performance (93+ Gbps achievable) +- Prepare both systems for multi-node inference and training with NCCL + +## What to know before starting + +- Basic understanding of Linux networking and command line +- Familiarity with network interface configuration (netplan) +- Understanding of PCIe and GPU computing concepts +- Basic knowledge of RDMA/InfiniBand terminology is helpful but not required + +## Prerequisites + +**Node A: DGX Spark** +- GPU: 128 GB unified memory (Grace Blackwell GB10) +- NIC: ConnectX-7 (QSFP56/QSFP112) +- OS: NVIDIA DGX OS (Ubuntu-based, ARM64) + +**Node B: Linux Workstation** +- GPU: NVIDIA GPU with sufficient VRAM (e.g., RTX 6000 Pro, RTX 5090) +- NIC: ConnectX-5 or newer (e.g., MCX516A-CDAT for 100 GbE dual-port) +- OS: Ubuntu 20.04 / 22.04 / 24.04 +- PCIe: Gen4 x16 slot recommended + +**Physical Requirements:** +- One QSFP cable (QSFP56 ↔ QSFP28 compatible, 100 Gbps negotiated) +- Direct connection or dedicated switch + +> [!NOTE] +> Interface names (e.g., `enp1s0f0np0`, `rocep1s0f0`) are system-specific and will differ on your hardware. Use these commands to identify your interfaces: +> ```bash +> ## Find RDMA device to network interface mapping +> ibdev2netdev +> +> ## List all network interfaces +> ip link show +> +> ## Show detailed RDMA device info +> ibv_devinfo +> ``` + +## Ancillary files + +All required files for this playbook can be found [here on GitHub](https://github.com/NVIDIA/dgx-spark-playbooks/blob/main/nvidia/heterogeneous-distributed-inference-rdma/) + +- [**test_nccl.py**](https://github.com/NVIDIA/dgx-spark-playbooks/blob/main/nvidia/heterogeneous-distributed-inference-rdma/assets/test_nccl.py) - NCCL communication test script + +## Time & risk + +- **Duration:** 2-3 hours including validation and testing + +- **Risk level:** Medium - involves network reconfiguration + +- **Rollback:** Network changes can be reversed by removing netplan configs or IP assignments + +- **Last Updated:** 01/23/2026 + +--- + +## Instructions + +## Step 1. Understand the Architecture + +Your distributed inference system uses **two separate communication planes**: + +| Component | Purpose | Protocol | Latency | +|-----------|---------|----------|---------| +| **Control Plane (Ray)** | Orchestration, scheduling, actor management | TCP/IP (gRPC) | Milliseconds | +| **Data Plane (NCCL)** | High-speed GPU tensor transfers | RoCE v2 (RDMA) | Microseconds | + +Both planes use the same 100 Gbps ConnectX network in this configuration. + +**RoCE vs InfiniBand:** + +| Mode | What it is | Notes | +|------|------------|-------| +| **RoCE v2 (Ethernet)** | RDMA over Ethernet | Recommended for this setup | +| **InfiniBand** | Native IB fabric | Requires IB switches | + +> [!NOTE] +> If your ConnectX-5 is Ethernet-only (not VPI), RoCE v2 is the correct and only supported mode. + +**Core software components (required on both nodes):** + +| Component | Purpose | Notes | +|-----------|---------|--------| +| `mlx5_core` | Main NIC driver | Kernel module | +| `mlx5_ib` | RDMA support | Kernel module | +| `rdma-core` | Userspace RDMA stack | Package: rdma-core | +| `infiniband-diags` | Diagnostics (`ibstat`) | Package: infiniband-diags | +| `mstflint` | Firmware inspection | Package: mstflint | +| `NCCL` | Multi-GPU collectives | Built into PyTorch/frameworks | +| `GPUDirect RDMA` | GPU↔NIC zero-copy | Requires nvidia-peermem | + +--- + +## Step 2. Set Up the Workstation (ConnectX-5) + +**Hardware & BIOS checklist:** + +1. Install the ConnectX card in a PCIe Gen3/4 x16 slot (CPU-direct, not via chipset) + +2. **Cooling Requirements:** ConnectX-5 100GbE cards generate significant heat under load. Ensure adequate case airflow and monitor temperatures with `sensors | grep mlx` + +3. **BIOS settings:** + ``` + Above 4G Decoding: Enabled + ASPM (Power Management): Disabled + PCIe Speed: Auto / Gen4 + SR-IOV: Enabled (optional, for virtualization) + ``` + +Verify PCIe detection: + +```bash +## Check if ConnectX card is detected +lspci -nn | grep -i mellanox +``` + +Expected output: +``` +03:00.0 Ethernet controller [0200]: Mellanox MT27800 [ConnectX-5] [15b3:1017] +03:00.1 Ethernet controller [0200]: Mellanox MT27800 [ConnectX-5] [15b3:1017] +``` + +## Step 3. Install Drivers on Workstation + +Check if mlx5 drivers are already installed: + +```bash +## Check for existing Mellanox drivers +lsmod | grep mlx5 +``` + +**Option 1: Ubuntu Inbox Drivers (Recommended)** + +```bash +## Update package list +sudo apt update + +## Install kernel modules +sudo apt install linux-modules-extra-$(uname -r) + +## Load drivers +sudo modprobe mlx5_core mlx5_ib +``` + +**Option 2: NVIDIA MLNX_OFED (If inbox drivers insufficient)** + +```bash +## Download from: https://network.nvidia.com/products/infiniband-drivers/linux/mlnx_ofed/ +wget https://content.mellanox.com/ofed/MLNX_OFED-24.01-0.3.3.1/MLNX_OFED_LINUX-24.01-0.3.3.1-ubuntu24.04-x86_64.tgz + +## Extract and install +tar -xzf MLNX_OFED_LINUX-*.tgz +cd MLNX_OFED_LINUX-* +sudo ./mlnxofedinstall --upstream-libs --dpdk +sudo /etc/init.d/openibd restart +``` + +## Step 4. Install Required Packages on Workstation + +```bash +## Update package list +sudo apt update + +## Install RDMA and networking packages +sudo apt install -y \ + rdma-core \ + ibverbs-utils \ + rdmacm-utils \ + libibmad5 \ + infiniband-diags \ + perftest \ + mstflint \ + ethtool \ + ibutils +``` + +## Step 5. Verify Workstation RDMA Stack + +Verify kernel drivers are loaded: + +```bash +## Check loaded drivers +lsmod | grep mlx5 +``` + +You must see `mlx5_core` and `mlx5_ib`. If missing, load them: + +```bash +## Load drivers manually +sudo modprobe mlx5_core mlx5_ib + +## Make permanent +echo 'mlx5_core' | sudo tee -a /etc/modules +echo 'mlx5_ib' | sudo tee -a /etc/modules +``` + +Validate RDMA stack: + +```bash +## Show RDMA device info +ibv_devinfo +``` + +Expected output: +``` +hca_id: mlx5_0 + transport: InfiniBand (0) + fw_ver: 16.35.2000 + node_guid: xxxx:xxxx:xxxx:xxxx + vendor_id: 0x02c9 + vendor_part_id: 4119 + phys_port_cnt: 1 +``` + +```bash +## Show adapter status +ibstat +``` + +Validate PCIe bandwidth (replace `03:00.0` with your actual bus address): + +```bash +## Check PCIe link speed and width +sudo lspci -s 03:00.0 -vv | grep -E "LnkCap|LnkSta" +``` + +Target output: +``` +LnkCap: Port #0, Speed 16GT/s, Width x16 +LnkSta: Speed 16GT/s (ok), Width x16 (ok) +``` + +--- + +## Step 6. Set Up DGX Spark (ConnectX-7) + +**Fix repository signature issues (if needed):** + +If you encounter GPG key errors: + +```bash +## Remove problematic repository +sudo rm -f /etc/apt/sources.list.d/*ffmpeg* 2>/dev/null || true + +## Download and install updated GPG key +curl -fsSL https://workbench.download.nvidia.com/stable/linux/gpgkey | \ +gpg --dearmor | sudo tee /usr/share/keyrings/ai-workbench-desktop-key.gpg > /dev/null + +## Update package list +sudo apt update +``` + +## Step 7. Install Required Packages on DGX Spark + +```bash +## Update package list +sudo apt update + +## Install RDMA packages +sudo apt install -y \ + infiniband-diags \ + rdma-core \ + ibverbs-utils \ + mstflint \ + perftest \ + ethtool +``` + +> [!NOTE] +> DOCA-OFED is **not required** for DGX Spark systems. The standard Ubuntu packages provide all necessary functionality. + +## Step 8. Verify DGX Spark Interfaces + +Verify network interfaces: + +```bash +## Show network interfaces +ip link show | grep -E "enp|ib" +``` + +You should see ConnectX-7 ports like `enp1s0f0np0`, `enp1s0f1np1`, etc. + +Verify RDMA interfaces: + +```bash +## Show RDMA device to interface mapping +ibdev2netdev +``` + +Example output: +``` +rocep1s0f0 port 1 ==> enp1s0f0np0 (Down) +rocep1s0f1 port 1 ==> enp1s0f1np1 (Down) +roceP2p1s0f0 port 1 ==> enP2p1s0f0np0 (Down) +roceP2p1s0f1 port 1 ==> enP2p1s0f1np1 (Down) +``` + +Check PCIe topology: + +```bash +## Show GPU and NIC topology +nvidia-smi topo -m +``` + +This shows how GPUs and NICs are interconnected via PCIe. + +--- + +## Step 9. Connect the QSFP Cable + +**Hot-plug vs Cold-plug:** +- Hot-plugging QSFP cables is safe with ConnectX-5/7 hardware +- Cold-plug recommended for first-time setup + +**Connection procedure:** +1. Identify ports: DGX Spark has 2 physical QSFP ports with 4 logical interfaces +2. Connect QSFP cable between any available ports +3. Cable compatibility: QSFP56 ↔ QSFP28 works (100 Gbps negotiated) +4. Link detection: Should be automatic within 10-20 seconds + +Verify physical link detection on DGX Spark: + +```bash +## Check link status +ibdev2netdev +``` + +Expected output (after cable connection): +``` +rocep1s0f0 port 1 ==> enp1s0f0np0 (Up) +rocep1s0f1 port 1 ==> enp1s0f1np1 (Down) +roceP2p1s0f0 port 1 ==> enP2p1s0f0np0 (Up) +roceP2p1s0f1 port 1 ==> enP2p1s0f1np1 (Down) +``` + +> [!NOTE] +> If none of the interfaces are showing as 'Up', please check the QSFP cable connection, reboot the systems and try again. + +Verify on Workstation: + +```bash +## Check link status +ibdev2netdev +ip link show | grep -E "enp|mlx" +``` + +--- + +## Step 10. Configure Network Interfaces + +**Network Configuration:** +- **RDMA Network:** 192.168.200.0/24 +- **DGX Spark:** 192.168.200.1 +- **Workstation:** 192.168.200.2 +- **MTU:** 9000 (jumbo frames for optimal RDMA performance) + +> [!NOTE] +> The management IP addresses shown in examples (192.168.1.x) are placeholders. Replace these with your actual network IP addresses that you see when running `ip addr show`. + +**Option 1: Temporary Configuration (Testing)** + +> [!NOTE] +> These commands are temporary and will be lost on reboot! + +On DGX Spark: +```bash +## Configure RDMA interface (use interface showing "Up" from ibdev2netdev) +sudo ip addr add 192.168.200.1/24 dev enp1s0f0np0 +sudo ip link set enp1s0f0np0 up +sudo ip link set enp1s0f0np0 mtu 9000 +``` + +On Workstation: +```bash +## Configure RDMA interface +sudo ip addr add 192.168.200.2/24 dev enp1s0f0np0 +sudo ip link set enp1s0f0np0 up +sudo ip link set enp1s0f0np0 mtu 9000 +``` + +**Option 2: Permanent Configuration (Production)** + +First, identify your active internet interface on both systems: + +```bash +## Find your internet interface +ip addr show | grep -A 2 "inet.*scope global" +ip link show | grep "state UP" +``` + +On DGX Spark: +```bash +## Create netplan configuration (REPLACE interface names with YOUR actual interfaces!) +sudo tee /etc/netplan/99-rdma.yaml > /dev/null <": + password: "" +EOF + +## Set permissions and apply +sudo chmod 600 /etc/netplan/99-rdma.yaml +sudo netplan apply +``` + +On Workstation: +```bash +## Create netplan configuration (REPLACE interface names with YOUR actual interfaces!) +sudo tee /etc/netplan/99-rdma.yaml > /dev/null < [!IMPORTANT] +> Before applying netplan, identify your active internet interface to avoid losing connectivity. Interface names may change after applying netplan (e.g., `mlx5_0` to `rocep1s0f0`). Always verify current device names with `ibdev2netdev`. + +## Step 11. Verify Network Connectivity + +Test basic connectivity: + +```bash +## From DGX Spark +ping -c 4 192.168.200.2 + +## From Workstation +ping -c 4 192.168.200.1 +``` + +Expected output: +``` +PING 192.168.200.2 (192.168.200.2) 56(84) bytes of data. +64 bytes from 192.168.200.2: icmp_seq=1 time=0.xxx ms +... +4 packets transmitted, 4 received, 0% packet loss +``` + +--- + +## Step 12. Test RDMA Bandwidth + +Identify correct device names: + +```bash +## Check available RDMA devices +ibv_devinfo +ls /sys/class/infiniband/ +``` + +**Device name mapping:** +- **DGX Spark:** Use `rocep1s0f0` or `roceP2p1s0f0` +- **Workstation:** Use `mlx5_0` or `mlx5_1` (or `rocep1s0f0` after persistent config) + +Run bandwidth test: + +On DGX Spark (server) - Start first: +```bash +## Start RDMA bandwidth test server +ib_send_bw -d rocep1s0f0 +``` + +On Workstation (client) - Connect to server: +```bash +## Connect to server and run bandwidth test +ib_send_bw -d rocep1s0f0 192.168.200.1 +``` + +Example successful output: +``` +--------------------------------------------------------------------------------------- + Send BW Test + Dual-port : OFF Device : rocep1s0f0 + Number of qps : 1 Transport type : IB + Connection type : RC Using SRQ : OFF + Link type : Ethernet +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] + 65536 1000 11664.71 11664.25 0.186628 +--------------------------------------------------------------------------------------- +``` + +**Performance Analysis:** +- 11,664 MB/sec = ~93.3 Gbps +- Achieves >93% of 100 Gbps line rate - Excellent! +- Link type: Ethernet confirms RoCE v2 is working + +**Performance expectations:** +- **>90 Gbps:** Excellent - Ready for production AI workloads +- **80-90 Gbps:** Good - Sufficient for most multi-node training +- **<80 Gbps:** Check MTU (should be 9000), cable quality, or PCIe slot + +--- + +## Step 13. Configure Environment Variables for NCCL + +Add to both systems (persistent across reboots): + +```bash +## Add RDMA configuration to bashrc +echo '# RDMA Network Configuration' >> ~/.bashrc +echo 'export UCX_NET_DEVICES=enp1s0f0np0' >> ~/.bashrc +echo 'export NCCL_SOCKET_IFNAME=enp1s0f0np0' >> ~/.bashrc +echo 'export OMPI_MCA_btl_tcp_if_include=enp1s0f0np0' >> ~/.bashrc + +## Apply to current session +source ~/.bashrc +``` + +Verification: +```bash +## Check environment variables +echo $UCX_NET_DEVICES +echo $NCCL_SOCKET_IFNAME +## Both should show: enp1s0f0np0 +``` + +--- + +## Step 14. (Optional) Configure GPUDirect RDMA + +**When needed:** +- High-frequency GPU-to-GPU transfers +- Zero-copy GPU memory access +- Maximum performance training workloads + +**Configuration:** +```bash +## Install nvidia-peermem module +sudo apt install nvidia-peer-memory-dkms +sudo modprobe nvidia-peermem +``` + +--- + +## Step 15. Final Validation + +At this point, you should have achieved: + +- [ ] Physical link detected - `ibdev2netdev` shows "(Up)" status +- [ ] IP connectivity working - `ping 192.168.200.x` succeeds +- [ ] MTU set to 9000 - Jumbo frames enabled +- [ ] RDMA bandwidth >90 Gbps validated +- [ ] RoCE v2 confirmed - Link type: Ethernet +- [ ] Environment variables set for NCCL + +Your RDMA setup is **fully operational** and ready for distributed AI workloads! + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---------|-------|-----| +| `ibdev2netdev` shows no devices | mlx5 drivers not loaded | `sudo modprobe mlx5_core mlx5_ib` | +| Interface shows "(Down)" after cable | Link not negotiated | Check cable, try different port, reboot | +| Ping fails between nodes | IP not configured or wrong interface | Verify `ip addr show`, check interface names | +| RDMA bandwidth <80 Gbps | MTU not set to 9000 | `sudo ip link set mtu 9000` | +| "mlx5_0 not found" error | Device name changed after netplan | Run `ibdev2netdev` to find current name | +| Permission denied on `/dev/infiniband` | Missing RDMA permissions | Run with `sudo` or add user to `rdma` group | +| GPG key errors on DGX Spark | Expired NVIDIA repository key | See Step 6 for fix | +| Lost internet after netplan apply | Wrong interface in netplan config | Identify correct interface with `ip link show` first | + +--- + +## Next Steps + +Continue to [**Distributed Inference Guide**](DISTRIBUTED-INFERENCE.md) to: +- Set up SSH and hostname configuration +- Configure NCCL for multi-node communication +- Deploy RDMA-enabled containers with Ray cluster +- Run distributed inference with vLLM +- Benchmark performance across configurations + +--- + +## Credits + +This playbook was contributed by **Csaba Kecskemeti** | [DevQuasar](https://devquasar.com/). + +For a detailed walkthrough and additional context, see the original article: +[Distributed Inference Cluster: DGX Spark + RTX 6000 Pro](https://devquasar.com/ai/edge-ai/distributed-inference-cluster-dgx-spark-rtx-6000-pro/) + +![DevQuasar](assets/devquasar-logo.png) diff --git a/nvidia/heterogeneous-distributed-inference-rdma/assets/devquasar-logo.png b/nvidia/heterogeneous-distributed-inference-rdma/assets/devquasar-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..e90b1efd3d7539398171af5129a7f9072b5a7a4a GIT binary patch literal 36156 zcmeFY)k9oMusys72=4A0T!Op11q%ds5AGf&!QI`12LizYBm`%0cXtcUK!VSYlXLF9 zZ{J_=z0Ax*Pj}T?tGcRt*N)OqQ@}taMFju=LrGCq3jh$Ge}V##5uU%=OCuct01PO} zN@@Epo#ff(rtZ#0)MhyybNo;_bILGzAqP5?>s<>-pod)9L`6Z;GusZM7~UNlTO#3W z`r|Ou{`|9+z8YqIG;(h)CFOIPWxHn&4|51Ux30}mLaW3|3DrcUd-IhaZ z>LBr&4V6x4g&A50gTgD#e~0NEc3Vh?cHFUAGYH&kBb4xj&M)|kTobzSNd1)mmj?@) zPC9K$Yqj|Y3Jd<`sTo~b>bxh>@qEizSawGx?_>=AUu$Uz-80YGL%#3T=8oTTj0SBQ ztMBvXTlla;f-^aD4vUGH|57Ww2#=Dy(RU8q}?QF4R1ojISOGOS!?zLH}nm54)oOnanZPDq*rs92e75iNBe*6r;C~YvATpM{K!Dj_NVNBPTJxJSaLS@ zgfegQw5uB`$VswLv_{Z3?J#^cPJf8z{VDz5IL0|XFdehB#CAlZhb%%#;`l2)MY$1O zI#|bF;YOMnohBx-tYQOkwS{)dx0o)tCTf_fy&Hzq&eriLo8_K+^QZoZF|7&T|aii^>W|1htLriXFut@Y{I=y73 zOyJl!Ewq7ACj`-RpLP9Lgzj1ehfSnI!F3u8 ztc+dHjMAFX_{igfyokH1j+F*(u<&duOCMnCm;frC4QqMD=D*RhV?}-cjxUvHf^E+d zff#R-Y3>mk(~{!7jt#QU9YZ@-Te!h`(-lO>jb(kgZZNJt_G;$81-X;EgK$)LaUptw zY8^bBLyToWj3sP+ezHOzJ(w zgz4P^w)OM;DA;AmyDKUR{?J$7maQHlDt83ZFENY_9RQ6?&SgQJ|NUPGbF>}DtCGqv zy4}mqad6bCneX@)>>lQCV9SOr#ccWfp)9Ku9;FEXqOXmRa1=o1G+;YDsnvdun{S&i zsym-CIdw(imejw~qUNxl0;{ugyzU{H>u7^A?33(_6F$k}ik`ZeLq8>1Gt z{`~Q&kw4HMH*ZGhFWN7GN}VWljyW>jJnJ-Nmq_W`Rf-7kZ>rKQL#>G zZl!RmMi*|dK=*eX%}YR(zEdAU{D-vq1xO0#)vYT=VRi=FF6;v#dhA^uZ#Xlu;$K4G zsP*jzB8gAgQySP`fWoo{U#@=vRd0HIW{)a907(L3|7RuJRV)pF42ND+SGrAj>r5rd z_o`uR>J>K$B_DuXSoab9Klp?ImKMkW+<_GrGSdip!Sj537@yf5Y108}uy#1>|E^3D zx$2YGr%zb$Y~Wo5r6gai-W3e#KxLV@W0_8w{>8tpfHu(l`|ts64_Ju3#l^vU19-i@ z9BqR;ISTnCf^8o%u1sw&-2c$&Sf9?~LIgmbTQ6h!Uv0hkeY>SrwrtU`w*l zRhFyK{tNnsOEzb3_ui}{MDIQ7bk(^d4R#+G_N7{~_a%m6H5VLGKyp_s}I3zGkB!o3W&bnLgV|M)(Mu z_$WoV+|Tp^^p`?K;3lgKy*~4&cn()ObT#9=vEKnmYkMd3B+*%!4hIIlPs^Ek@B+k9Z1HtR)L!5A?&+fx~hjM5v0zk5a^aL!m6%g}#WD-6Lw@C27H zo-GEwz5Kp=6KvOYm>!I$ft~nQkO<&ldq}6?X(fkkZ{4id_lRrYEo2jx!1&Bx)qniO zQ^hxrQSzND{Nw|)VO>4B-|f+P#63%9d?tmk!xC$|6b z_a*Q!#@v_y&A70J`$(ed{>Y)LUjsu+9BOs5g;w?>r@`a>!^kr#WQXYL)K)b4s*)zv zpx&crN8u-M;9@_u3&;G2M@OUtfJ~jS>E<_34y~;6X%*&xN#rUoBaAJj<^N*)kJ%^M zr)TNnBlZy>cqW!Jd7n^jU&G2>O{90*E{UyM4eI|3ch|(&P&hF48g=2K7tB|`=3b!qr|GzMRYq{Y&D{{^bZ-dx~FyAYR_w?Ydi~-=>Jp_)xN{{z=qhL zcDDAD58TwItV7t(KutCi6IvwJjGbEP5_NXa{zSOaLQpF z^pAK(6rlf`FtQ8<2>{K+@H_{lC(#@*cZoXd_ITOYq5`If``_RV?5V_Ga7_hd>~ZH` z#rTCk#;=(Z)b<_1)OX1Z-+i}nix-al4?C&>D+Hl$NeJ9nPQI=bX-&cgSX2YX$z7S;;np zN(CfDp3F7gCiX0APvI#MHIPOFmRY8g12EeL$!-aDGDYF!v2QGVLATJ)ZNr&XdK_=q z_;>k8JS_mhUePw3JJN(VmWNgSVTbc$lpEr6Zh>fX)DNDU3_EB;9v$wL!F_Y-*%P5% zfeC3N7O1B6JRcm0{xZNA2_R2pI>(w(uU4vOGtOh15yveM?$`c-!pS)4!&mHGBRs&o zh&NpU4X2Wc#OcF+_Uc_82A%>ZH)ZC(nZPX<4UC^X6_b>!W_u~xULSuIZwM?jBYsVv z$K%X5-k%vE7V4oS%cy_%ATZ4AXB@GCxBlh%ukrPOVlhFGP|Ev8x?x?f zsEFxA3OATP%XAC16ztw$O|cVZ8nHQZ-Is!%EXOG9&Lwj5F(GSA7_;)|Z-U6RgQJf7 zcD2uTHx=$_n)Srz`ALAP@A2qSbXemTqwCW<6I&g)4wtUO{1?EiJXpA~n0>!KCQQ|e zrAnv*lj$EP+#b-tz#%?oqQ`*@npTi?3FT{k+e;<5LAG+vI4EAr4?dJzSW&?~NV2Kp zk=I}#SG zlqX;XhdDsul2Q>D9kCkD@J*_^-e=+NzXFx|J?K^Z9i@{Ae>%$1{*MG|goZ8!9mXu0W+LXq; z7!S$b(X)kzojsnK8Glc8f|Ll7!HBos4Yg76S84};=tbqCc1?9IJDx`uy?{@t=kwla zd$~aNUxk33=Db6{J6+i$go2vEtlIRDBl}z*P^AJfu%Z8|A~$kmFpIR(U10bWQ_(ur zhEmKw$-Hj}!CHkI5uiu{{sOWkhz$%gDNpTBCyAp_cKsri{B#RT5de^-M>M%L{_!Mo zTM8Jlc26et&dSZ(;1TXjA&qXcZ!Za1aog7R(3OR!gNb6YMv$r#VW$5?^41MDfNjG%elF}J7>9FJ z{Ca8t%r<6WAsud?S*g0aLfqaOKVPAG@GgZFsE9rZ9JI2ji>8%>q5Tz*I$R+HTfCDy z9g4^<6vrc&$8UA>gz|z!`pq{+&sf@g8@m}kaPKyZCM3yf7@I|wW#o*voGd{{$41#`a1>h{xYZ~CB%cLigV0W=~h0=pDO zEHIF)?-c0cgsl>#SSk@J%c7Nsn2V1QuribxQ-n*B6p!_50*uI8j=^pxWm!MS5Q9FY2tOp$3yNMGMVzZo&pX~Kq+D`aJM*xU zg7jDK_NTn$>Wjm%+OsqrQaqhIBHM-?NkgV~{4}E&^kd zb5>hZj=-9{B-FyYAS^nfnpa{KD(zQ=~m3ateL(Vg3!c|OZhc2A1(8E#5?-SYbCCSng8l>#9 zk?)ijA^zf)!mNuyM&irB>w^KMTxrIoiAe2_n)xbxv&XiqIeOiJgmt@uztbpQl}u)H zpl?_iOotk2At&Qrut!uf^wN0OCHak53FY98rhgr@;8LqSZ5Dcwt0sYJCx5!>HWpaB*~41!W3e}5#6pOS={0xPdbIad z8P(m#_?}^ zUk#Qo2jhOZsyUW(eAR0X1OfVE{eeTu>niy`++jC%_(W7$3H?=i6|*H8>rHB9^>6Cc zKyYqkP(?O8SJQWNe^<)lL)H9EeSGfZSMll~*G9$)iT&w>^pd-sJ897|bzI8oBWPeR zAJ{8*_%d{Tx3ujb$7y!<86zt0tXXoc-kPGHK0BfuMIu=!&A&N!t2ZszENSSu#65yh z!2pTprsG&C<05}79L;aeh>WZe$_T3iSIR=?UiJP<5mRL_Ah9Fg^X8u7OmM;Hjr}ka z#|A#>qDE3#bX@Gd7oh>?ipV8#9Btrl94vZA)e@)&N6f+7J5Y?63+Rh)>>Q4G&Nu}} zfX?)QE%N?kMwuH~cal70HgJfH?Mg@N6L3x3Ih(7-x-4DTb8O4(!V`bpeiel@P{K&$ zMo8h932he|dbJY-sqfRVUrl7B9V;4cfOgzA))#G>c0U;;n2eq|Wdc0D5F~XLReF(|$O3mQtsS~cck1e41$kiT z{){?}|GDjm2z@oD*tu$ z5+DnG!;ODojW+&id6+e}DE}3CjFf>p>l>ksfxNd39~en~H3$hR;?#*(4 zd-`xSsq&=Dw`Kcj)=SLoT8PKDV&G5h%R=MVIvuKa6Z^RfnciU*ETnEBHBkorWd9&? zqIXz#XxfD-zO{d)He}Fyn%sxH7EO(O;zy*?Qc$D$sQexe5X$Y*i(u%>j&gg^QIdAT zH}U3{P6ub)V8|z^WeL?*>z(i0SHfCf=p`|W{ldhaJ>|A>Wyi4e-NFT)umOGaT_Nx9 z+`~lB5BmYTaJ+jAdm5YPKTr3V)vlGaB@+o(MtCt>&*Q5WJI|QD2BkLZ0wohMw#?sl z)&z?XI&sFm8J)Lu<}`j~p;zot@gJ)oyXvzunDQkoO=!Ry$+y~;1q&@_b>`(dB8Om` zlEwVI5%^(j*HqP#zz><*WdvzNNVUlpp-FoY>Morkkq@c$2(6dqpDmfAs*}{*P!kDJ z8~oyoB%-(m<{l`_*!D?jALxMxT7X+H4fb_-_VyiPaA^Y_>xa6*Jdc|Jrk^MGXmv>n z13rWJ@s4>j`=mB477xKcale9(jIz46p3zzv;f^^nW-I{&xl_G2ZLq96Mae#b3vs9d zkA4YEji%2g+lp~NHecZQxUu-vA6t^5l3fH1Q%EF%s~3&zi4h-Zfsc**>W&_6ug?hX z7PI|Z#J4f_mm!ktO?_~4#5KMgqZGdV9X`}M?CzcY4Ojfa&M$#B0^qr@#Lf}ToUF?1 z=cBB4MLZlZHA)vrQ{6Vgniu6J?ybW2Gs`32@beeqxXP{@sR!&Ju)eydU^p}|-BK%S zGkH4bl)v;b*JT*&b1NX4JNq!1o~z)bzx&}3M&mO*&d8NExOp#;r7Lvc4=ks6#&5u1 zMgRjl(&9X!6N1}hhJ9FeX2o1knUIL8==Hmrslm~3>`?5ECLtn2h78UeHgp$xqQv*8_gh?Bv~u?_o> zw&gQL)u4x5o0W$1^tdx|Nd*&6zNbGqXTFF%?}GYqfi+|>;N&-Toi?#V5x^t3nD#!a zFOTCmi0e-X*5Sz(aVkWP?RW3ywUfel52iA}3P&x;C>|Mrp$NBO2(N@G*7E3DMqMtZ zw7={*8Yfr~(3;_3Z=3R`E^ZqnndWHQZ9Xkg5pH{%*Ssf@`qHsiS4&<<$BmD^vo&ErSZ-m)eHd_t*t^48p5lS!YjRx$-s!(FXzM%oZIx9_TnYig+Q#0)Q zCBxlV*vXZQkrIcI#p{bU0yD+h#_zwGELS#q9h2zRMf9oKkg_sq^R2fqv|)-4$fcK0 zb0RSpWsW-9P(?89Vp}5&Qm5SU7Lz#HDVmJ@6pYBR>-xz?6F8#ZMjLZ2rlEzysKd!1m_at;i8DbVO3{rpEMszRh-`V=3>gJlmr!pUkh&RB0Ys8Wt&cUHJp z!mMg@WjPYbnCStDF2R0C>eqrVjusL~%*CnhTF=F)Bk5%P+nr>xH?^n}Q5}um(~0uDWh8!9DNdUQAjC^XXWNvJ zKSmK}9vSpL%0l;xJ&4{~ia7U7NLmAhm|g?Llo)pMbnXKUR%Zw#odq|PN5-5mmre0( zy}yiXYxpRg zF+7Jfz!0WoI>+7U&uDx@XUTTmk^$ddz=NkjwO-YEEPA9i5z7L?m@Z&%&QWr7dw5P zJ|NT^c&RmIJ-Fj7x;y#6X(y5))B?qW>4SH~>Jm0Nuj!`ko|GA5oT_6f*eo$s7JD%< zwSLFVQN0ZFN<*R2Lg2LR)4Tbx6qR6kRB{DAxAJW6ukz zwZY{67shL5KTnn|PEg_MbLPHf096)91+G{jb~_(TUYCmaKpCjcu^@uqK5f4ZI37bl zi+MlmxJlq#goR;|iVmu`A6|&`F^0v=Ee;XBbAlmIK zhDO9(Zcp(@f?vdhe9c!H?21~hv_ruHZFoh2)3*zK$T@o-dGzg>MT?{3MEkF=N=!z+ zNTlrLb`|L1_vCQwoLY8&#fRJRM%GXek9Xs(wh=jl(%nsFvc-VxnP-X_n=kkcEzl{L z_ZT9i4dKzPl~{z7CAqNmE*i!LnW+EEt{T$>f`Qde;Qu8(S}wh&UuvUXj*>0Ke|PVN!Oty+Mf z#a@-LI3|+&yNS5eO+t7rXCG$!;HK+5qT+Ixsul;!S9+*P{c)+&r3EO>Z#R)A-+jA$ zESVc|A|dm*vJi^#3w_cYCYn`4Jh{OvfN01yXOoyZAwuN=D-ZS9Js8cNtH95}_@k_n z>GUJN2r;g#A~Hdd&Fe(AHNIR2#*n~CmLJ4Wf*?hi9P|P?!V#D|p(|B+?`;{+z&-LC=1aPo0JG$Ub+zkWO1m5h@Bsl z*ssE607nz8KaBs>r`{T!lq;*SG?~mmxoDM6dVghUBEz@rm%mBr^d2p_M`>kD#Iarp zaUlg|z&YiX#IXK2d|}k;)Ldhv-S+;h1P!Zxi|kFWygb8~+jTIJg`-~@+m3Ccx`r8| z{ZNx?6bo|IZ6s#Ii%!z{7m|qU3CvXT^w1khCtSxksY}6#I<7DAf%YLdt|6O;y3oW| zpFu0;idh;EEGI6^@7z2#hlXAIfeGFy;yKi>TR&848o<bJ!=)U)Ur05hG}c0j-sa5F2agv@4FC-QR;uQW_&~d2RvJ=?-Hd#<{>iuxGTEo-bsvoT@U7PQ$v|?iywd1>im1_noinc70AV7zio_hn z`*(ftnhtRbT;2V2z>+)$OsH4%{ir<`lEhtdhr)r??D5WvaDU=)ikCSifeBa(*^<(g z*ufXZl0&5^vrtK?oD&)ew7q!z)$?${NZ00@4a z3z$P$5Iz(nnOm>~i8D{Hvx(K$azxKWHvda8NWG9631`CFcK@wPQ8HU#;=PSK6@2Bs zy}@VdJYTSmyvt^62|iFRd;K^=9ICFdG2prYM*4&fSr!cW&4C6U3#9A`Z|hhRXP(o@ zST3#+Pj!9$(U#<0ZeRh_*DwGzf!IHX!XX#L5nZ=qj1pX&2Ja#)X?bc}VBY>gukC{ zBE_GgVz`rfeA1IGFi=$|jXPik^7LI2e?nWMdjf%CWI z0q_rm2&pbYKt;B+jjIIw08|jdmNq8Zna%V;zbG`Q-bACojSIQq%gWZqQ>G_}b>0O_ z8TWz?XaYQ6jC_Vo)-u?YSc^XvotA+F6XFk}W zWvojeuWPnCe&No?4F`k*%p-K#Yp&4fv>lqU=g{q31VePd*sm5BXpb&)-LBa8&oJl8 zU++UIF1Ni6N5`~JV`wkGc@%HkYs2LZ!EJjq|6mB<1!3=C;ndm+1Q+gJm_NYt?lur3 zJ_WT6m{Ik5l}{y+-BOV;B`n6AHd7a}vLOIEbqR+n>cRc2ZIH|3ALnMOLTKN(*beWc zP<-&p<9Mr%;Oyre`Yh{rshiUk#}TM z`LeXV5~x2mNh%={~Bi47VA*)P&ng@GyGHyQm^yEC{#~9mbh;jO{2~LY-KB@9;&}MH`~D# z${;Wr67Z1h#*|?5yQ{?36$JR^({SxydnbDeUr?->QCM>gf)BsF9Bhc`=*J(@9^??; zbiTW%#quLH_5;sy8T;{#3%>1M7P^yT|43;eL}z6G7!>!Eh}x6R9`GEQo0{D&(rnfh zXyY#>YngD(sP-8>DU9QmDHw$g{=_&(XSF3C;f;O7-?=2_77e?}ec~7>1cKDnw02QW1 z>B&xJ#UjDKx1uj+JC7jA;WK+@gn28EL;V7VV!9go-b}M$Bu?63`ni zV&XPSI!W?~Iyag*NfpT)s1=+=GJ(~`!F0IJIu~r-XwnK)_UCAxxn!}Ho#F^xS{vQV z9uoj@S@^qwRYm~}AT`VU&8*POWR03gPZPdy9L(RG_0)-*9a=k+g z$H?8bAXWr-am{`5R09xh^SXo>9Mt;*q%`!J>X3c_P%+&yYe9ov@Qo`(lJft~U#>$Q-Yz1a$! z?sDkYJX>OCL(RzE!4kCQ^LJmkH>f49cpb(BNP5!^RAEkmy<%6z?JoKD!mBL>yM(M< zj<(uUO|Xv@S=hhbQ-D>##o=wAdj&E}02DX(A)3MQM}F6EMm7~PCf#Y@ za<{!hq>TCsjLA;8eHXq$KYt8x!r5lea`r5LSh>c}&~LL(^eb3C2p0GPI9AaE&AGOY zwQ?J*HQ&A>x;7#cn+Lc>_L6GoI`8VtQ8tZ!OSKLB=9)4#p77aSZ;W4j=Q-F0t|C0Y zdj!Y6SWLK{__rElJV=Ei~}O4CYpi zvtG#p7iD*!q5Bw((dRd+rRY{CE8ymqyt%cW`Eo|Taw4Tx%lqC8{XP+JRRegr5noVk z31gjp7kYyMxh>|;?9CE2_XM>bF8qSv_B5%MBw_5gfTeqcY$nG-ngi?1sXF=9yySox zo7upF!D-hd#6VKrq%TgQqsRqefmYLb%f4X*)#kK5{yVKaSbVw{gv0v%zNcjp7=cZ7 zFY_o@#-}P~K@NXl1xcPx64_Xd*?;T~2h(=Xn~Aycgy?`VuDhkxc&x{4xH6KFD?b#B z8TbU;z4fPn(U4=TL(zBVea@Cy== zmI!;<`vc{aKzTF1;sRzmG2GCeHBSDVyc(p1PvI0aymQ@*bB?rdrES&uRfO_s z=B@GwVg)zpewPsf&5>-2r$OBDyFEloE39tgU&JXCE@t%#^G(?pI1&yPJ&2<=h>SJ_ z4$MB+&bnM8TgW-`1))w~PCX-U^y#-3k0%er`^@T^e1df0v@goc+l8DFfaCqjmhu$p z(bf+cw?D>Z!Cx&N)`yRtjc+-63cE?wSMpv5K_*t2X3EaLcQJXIFpcf_u^5Rvh_zel z4WjUTm=_IX=o1|+i!8{`m8?0%euFMAk4n^%o5z~N6@K950v{2m_sgFMj907O%&QZ=_DN4D`Lbp#Mb=hxDlek z3@SIAp9H=(afJ&l5ep`ihSxcj#I#EO^7z(;e^1-xRRokPRMLD+vkf%!*M&;hIDGY< zZ08IMV?Xy4^9+y$CB6%>&vZVILK0I94FLP$(7#8e!2AB`ZTL#0^~mpu19pMNujpj( zBk)|t!%)XRrbu4#wE5Px>}A%^fb&sWenp~C=zaIlG5piBiN4i5Q)xu;d-+7}X804n z#YZgBLO$sop?PU8t(MNj4W8Z~?RvQE`zMifI*2lO17VF3b>Zx$6po6WM66}tVpND$ z=ENkA(EI{artN6+_j>VSpNDv2E+eYfq20}u$M|y}B`RIMSl{NS5)f8Bd3ct(ey{p6 zBfh1kncnv5q1(hNM+h)be{m>p%Y5e7{&e>Fxj!ofg@4t36tW@UoBM%-aRPU8O;zFj zXuvlT7~j;z;j^lHBrSIrq}Q$n#lnnNOje`0!oR#V^zGqfZXJY^Nskpr(T1x4TLFw={jR~Jr^|E-g@=fI%=J%)owrfpjSCV|N)&9bR4zxZ~G zQIW@a!w~|-hZ>JtTg=O@MBeQr%NT!5)sjXDJAVEBq()&)5v1!uK;h`5U|z;4c9<{s zHuV>Q$IXbq{P&Cu;$vlm9mx7Hv?*Ix;WI(_}^>o7+b_>fR1kA-PMFkdG zatTYA1>7mOeQ7IYB@-c@=A1HiGVtz7LJz9asr3nYBfEHn4SkFz^$XvF#OGc4PMlsj zXm$rbq_MCvWRu{@r(f1Y7#6)t(KF>9xPV%@mzgK7_8W@(kMk_ZEhFRg0VdopCu8_Q zal0##F{7`AV&vl0(LsnLp?b_N^n#})oKu1+sTau!-`3tjX_}|s%(Xs0KB#|V)v>Qb z0*AQGIjR386E(!NjrFxrhD4Ysws~lX@?3w=3U5eEbM*Cl-T@++OKZR2kOH&d|T@VpNCZgW-o4MxEDmjiSKPb*_KWtaR7Ojk}B zx8G;+T?yv3HWZN;t;*;(wSA)N{C9-KuV-gZt;oxTPG2}#R{M))j0auG@^cpO(%wmp8+0P{0#ADTsZ4+ivL_BzGEq0u#@7F+^CiX5s z&4Qc@{5uR|yYTtu9})z-tfvadgkm6Ep2cveEg#2y(-Rg?ELT_ZSzE(@W43h< zo>D0<=$a?6_})Lfy59cjB2eJN!taFx+ek0={cQw+{H0;3G7o12rJ0Y;D82*VF1W;K zf2W(|jvz#weZhmrcr(2aDpvPGlN)&bm7XLtNc)^dU@`H?Sa#b(UiQH;p}$JHHQqo% zB(5sQt2I+S*){J6zp`Iquxzev+wXvRpIx@e-d;jep&v#B0Fqx)gWbtM9hnE-K4<#A z-bS535&uVgSEPBvA?DmZWW;mkVDoe=s7|?NCxMX+64FoGloZYP|6L|3yOo7}0)I8z0}i^;u^4c)9C;d#5cjOwQp^*8N3IS+ANRzJR1;H^FgMd;pYlGDp^3q=s! z+JD?_JrQ^k@enw3qsk+9<;R9@YJS>3JctJjGdUIb_nFeQ+K+Jmjtub&&}`~ zT1b2rG@Mm4D;=}e>tONUQ!-N--%&&w<0h~gT=5yg->#s<+UUaX&OYEWCK}f8L z3+ew{1bbdal_(XX`CRb-nO>Z1QOe9teF{FXAs*>ryRwAiv=I!{$wU7@W375&{AqqQ z+(>XWR{YfN__@&ZHxZX1KiOv_<8OO=Zqv;gr?1IBGp78Nwhb>dbNgltZPRwjiCHho zJk-!4HB$=Wn(~WONL59~oBNMrl|}oxgJQD?SrOr7O%E!Qq=)mS&QmuwRV27r`4ESOdZF$rY^%XMz0j=Af zbb476k8`A($#4DFegMFIZ70ej)`{sW z-K-C@JUEH{<1J%}poOJX^bIV?UHw>+yuj%e-}~2C3IZxYuTjQ{fBnAAlHVZ{H$I!k zkQ{M${aH$BO!xlnXrL)()DbP}Cv@SLGiaFH`CE|%{ML;!RSuwVV()VN%XkcVSKDGD zkG%0qQ;MSK+i(Hec$do-wxYszy@lbD3OB6Vhp1)pj%2*F6p~*t)W2Q@wj%!e3^45z z%%@9zY}~Oi*?fi0ZAIgz?6Ed*!mpe_P3JpsJt}SQxlL)ZJ%8ot7Ugb`dq!>O#tYB;!JY2+A>2jnMY19vPWCmuk6s|4mEgvhiitJbJuwYs7r(yv{aNwBJ|mPlLDfOy(-jK)QsIK?!|Yu@D)eI zZX}q%qTS`E=?{H+)3=p%sT{pT{t_|b$PxK;jw**8tpBC0l3Ti%l zX9HOL()c*m4O#yQkRj?EY4=Z)*&-B7HK|=#BwNWB>tUuBh<~Pfu1A%4GLuW5VA_`$ zN#SX1mnUhuf(&X}v6Rs3GT#I}2^ z1Cd-c)Tx%kTq!wTkT$9AZvP>Kqb4+*eB(vW9;{elc3ev#k}oR#8$&Ap%EkQVfx#LP5QU z3M>Jkw+2?6x|f-jPoMEoTasBtxt~BbH#Hp91fYqr^Fz$_Qvg_yjiO`FNH;h1ETvhF zZ`N)|a79RIb?&|>kqaC+h;&H0MQ;+LKEn5xeW=d-0q!RAM0MFInYNh``wj@vC%_&gUGCx*Y^+Rkcd0LWI%vKKhJmh1G4yL(9{GM& zUbka_y!C-jQj^X1>aiowFB6K6*L5aUA7>ezn8}Sl_l5XST+kyiZz$JXP2AL8!0s1S z`BN;fKAc{fla$mh4tr+{yRI0?m?_H5T=}kPl4}ApMa&;U%XT0qJ!oRT6 zF&n+(bRpszCxoY)&ph`H35l}7vqhtiZWC9-==Pr}=My_n5_htfrbzg}8Pi1*G(nNTom30J-^O2XnxLIgAtN|+a_-bOc z1g7+HTzU4n({1(5?t5oeH`y6i1q*_gMTrpx_gQK_h4k+#+R)d*s7Q(H#lseV4bz$_ zER3ru#4oR$s}2R_5E5V#`|DY1;az39n6c#1#p9Mw#x3%%sW^lyknh-gpXc$r@FzK+`lnSmmr!wnVrsCPV(AL8GcIXS`f!7GwLD4)NlwrQ~Ax98L@;l z2~Sq9g*PixB(S)nnGoJsLcngPhL=5+>Nq7jR%GRDzWm(noeyWafrB+Vif`bGwtoSF zSr^59K(5S{V~vPtQSqIHhzdz9xe@42Pxhh9G`;DLQOZjke+fvx9|Hh#3tia)A6j0X zn|!Q=!~h5yXyT9NeSA6;u5u+E$o1F8-(xbcZ)vX{#%X><9w^6 z*LcqGWA1^|N-q9fv+~<81L2mZCkIP!)xoI;m3;Fz`UCwr&+kqTM^ zGdfsKabl+m|0_j?1!VYIh>>Ye9Bu`79UDHADJ<9>%lsN$MyKg4CFj&@TQ0rtr!i07Q;m@Wqd_k_pk+6@yCSP#NQ3x;$$Taa->3p+KsJwhUzypdhx`%T8Ls zp9fsiNWI@;n8m!U^!9@KIhP(I+DXEzZTc%O^h(>D(>pMgP0gP+YX6!FugR5{SelyI zevEKF235FnUIZqi0=^Mv6XOd0jeHV&7Hd1!>3vPO8U?>=jO>Fh^$F)UmOjSZVy*_^ zK!bTbZUhG{N#G$5nAYuTA;=m#o&2NtDnhT4!*~tdAsH8x`wF63hT(NXh^8hTk%#i; zjChW!!DjvpX_y%%s=4s(YbAje*oBy1gz_2$qX=ij;$`YT-=@y4K&~4boiM~DIm8aS zp1Yyx3V34V9Cr%Omb_3~J#AmoF&9SdM=7kxz!egDLtp+3+Vw-3N-uDufo&cN1|E;b zW^J7CN-gzY$18YXmEYQo?m8&@Mj0zf1X+{(Kc3DqpsHYN*qiR|?#@Fu2+|$WN_U5J zcO1IAOS(h4<4__9qI3u%DP7;W-uHgL_yPOOoIPs~duG;pp3eiD2vWlP9Yivylv%LU z%axRT;mi5>+pV5+yX_Hd^5q_(+ZYhFrqYY%ddX#GU^?(cZ6^K0qD5;H4xDg&N!!Pd zL8zL;nBRR(!yR|tFuIDC*;+bKJ139xRdRJUg;%Ip64o-`#G+4@!_f)HZwPF9c<%aR z5kwi=I_tom-xL4JkiUa-Hcy@}lugF`p-vjy{xnTWLTC0Tz^H!rZdv1Hf5!2f*#QsD zkq1e3E1u5d#$dIlR4IHw=NC)$v(^dY)$zA}a*q6gM?+xv`Q-Vnmu?pgKAMRHSJu~hNE1Y|A80eWOAOq!WjqF;|FtjO7)i@FS5~R-+jFI( zYWjMqagcaR8!0xktkBy1vpBSG58x8kiJE>-qU{u5!2%6nR{0g?MEQYF?_sG&KL;@F z_Eu^PG)zcs|9*4gwNj>i{KthDmokWl1F>6z` z*q%&TeT*?q%@YQQ&z>j2NjPZr4XZ~X7f))X`(PKNjw^E0S;I~LMdc!}&Sl>%OZDe# zxd*PE8;sL?QnC?pOD;BpK^4N*5kVRfr0h}0@4R84d=)yRLOIqyU7_*%qmL@4?X`=) zFo7Cu%D)S;a?~L@IpD(5)_5*?3>D;;^$`{o-OY$g|jXy z&ph@fR<2dU*vI5i^yu1|nNaI=+@8#e0OoCqD0y=J9#_&cM(&Eg#V(_6=2%o~jzG>z z($fHJ@+Ve+GfAMntB`EUyVoj$r79w@&I)AG5<_uvTVk|>t*k#i^Vbx5kdPtQai)Zv zJYo9a#zu8c8vi~;@+f~=o%m1K1=S8P=4gXpa>-D;HZznhucjD`g&@V**+Y77W97bK z?prPV%TPE6btYUNXWi~}ed^ZdS40M9-nX07ok3#k4$utbf*CNSHL@eyZgu3`8|mu% zSd;Hm4K#&B3P@&LmODQmK@%iWr25JvsB6=NE6q{uC&i;n z1(>)%Cdr&>k=M8|r1hy*D{FcxLfE$Z>eP~5JRlpU8dL9rGcUCtDJS9QS9)xK1yd}I zwQQyb<0{*H*%bJh37YU7MlwYRL;v*UYr6O@4P`#dYNyD)+aX>-p#{oV%lAQ4PK; zlg$OyhI^fuRkFm7c@lvWdt_}@;r-}M50ps!mgx5<)>xm{h9sa>VJg=bLU_SlX56MF)L|_@ z-KA%ByzmJzOFo~`h&L?5?FRhfnRvIx$hfw#Ne{gBgl#+gz?tB&JdjGO#B4dS{pzi$ z0vf~DguMAwU7D)&DG|8)QaVObH8)u#`Rtnp@QGtMVW++jd9{9 zrotvIaTF)!k_0-Msu{ZtLm_M#Aq;@MT}bCW`^&)iU2x$%OYBU1Q~Xc-F4uy6x11R> zfh%MeR|Gw+ty+{uBBHBK`fq<}loFn2&Jg7a;q-`k5NF^CyfIF817(65`u|#5lgo3M zx{gBAg!17se9;mvzn#GrzqlDR53}(ye>j$AU$C=8=vR z8iiK-Wa%6)_ELT5CMF?0G(`Bt_A1AihHX z5erpYnt2Bg_;v1-oV5Cq60}0wn!edkMw_cEg2C=f5RvPCMj7 zWS3}$u9EAH-93B_+e6t`B9!Jh7CF;en9Mk5v5dg+1MBPjhw&?u12^UpYyBy5XSXMV zD1&^B9f(rq6>_5+!?%SECfZ`rtjj8TlP}^5vZKheAr8LDyA|DV93{E5(B)kL-o~#_ zX2L}tYNP7&J6xIm{(*B~W+ArhK~AJo^CGjozYeH7js7-$ctHq*M|=pX_XEVUuc-Z7 zz7l451U#r1iQ(?kES4-(DRP(;zhor+(p-Eir*;rq?z%Ciy2A5kpFz2=k2!PkDprkR zFHDsH@w7$}d($5aa&_*_D1|QWD-65mlhUvn6s{=U=2Ur~$lC64qHQS><5|%y5rgXiCGPxTroABws#*rJh|T zt$3jy(G)J6mB}X?LDrrk5}VTZn)&UbiF#s&sjivCiEfQAZf98{TjjqFC{@0PzSrXlj9HcyV0L>8h2ibtv6X}@3NJ_{aJv8f;O=^rljXG{R$7#U@g zU{0OjViJRA#7qW4*$wm0>$#VP76k{djj5Jr@y!t=cLNH|svxgj<0-5++~M-VNhv>u zgl@*4$d|f9sTlOLN)WIX_>ViMb0!G_qSb||uUqT0G_u8(W#RiW zh&D*bNGw6|7AYR@;#bV>{Tm5a+W=OcJ1jW9i|e2ycEf^mC+_v3>n_SC+GEOUo{c?R zN{cGUa73gOOZ+`nmMi9~=|}LDAED%4DZHPZh#)C07P*<|ts8&QYQDDueGV|WFIxBz zA?NWTH3ARC$cC^8=;Ueq|>&C4Xp z0JPjO+-vr04^wp~Sn<3#mFEjTg|&8Pje%x=C)`<~Q$8vD-S1$wk{e;)MS98*_v2^k z0f=|Zo`U(;eKKe6C3q9|fA3leatU=IVjoz@!^_1iiw#GL?UW!Z?4$P3yHSZueh4>M zfqZjYz@RXMS#?52S1hYQ+!zTM8ivA&TtgAO6$NZ$RT?zL-gbz3B8`p5_7ybh0fzSh zNnuy;w!id|7qI8zdDhQ%VJO?1dkgQre>WH@>W&y_=8tkME)3wB8?IaX;W#O(YhlK| z^|?1My?v7baAZg#6aS8DFe^~F%NjI^K0#O_4Yv4XQ3(DccCebE*BTeyoxJHwh^aI9 zPO%r%_Qlqi|K=mPXTJ=iM$HgA`WLPB5UkcBEcD-geMb~mDc^aScWnIc2b=TBXaqN@20plbs@g@~@u~SYLj-hG_EF`ozTFi_bV> zCdN6D_JExg*P}myyVLU?{AL_nqMw5I#|syxZ>KbWdZ-3Wuf!A`xW55Y{?(C_2iss> z3CC0PMrR;)%F|IHt(7nRi6AOv(R}K>;N_lnG?_N>neGH_Pbe#B38!@>Oq3qn+f9Gm z)0gn0Xh>=a>kSpnsQ|c8fXQ9IZ>lBO(8a&Q9nKQ?+VBr4U*!cDPR20%=xHJ{Kzy0~ zkcM+tGqZ;i_Pd)?ST}NSS?YBZ4QG;9f8V8ur#CasIZc^K7r{0EUrK9?CG@hZYGUy= z3}M02bhg7l3$u>52G;uTZTEIv8}s35i;{@G*q$YK^^A6Xr`id8*&WVzq;(A|5BWJB zE=JA(D>5ih}U2YGpOb_NA zy4i0?_;IEWebE`lrh4_8vMdM)XjZVUq!HOge+;l>I@^nYa6F(eiMiom?vppfv9UBB=N zmV!b>wwrwJV!B3BL}IB!;?b+gm@)^@nTS}>iQo#;(Vr#30=Z40(!$*w9Bwei25#|S z3O>RSy1;3J{%D@WBx;(CK8(;yz=-Tq!hAw~bJ|#Pm)H1Nv~Pl}MAsR4#33wolEZsY z*f4ZXY}lVr5n!(s<&dg$Wm0xZ0UmwQnZ(wa-wojg%@c?X^XSnlcsMI*wKhduN2#wr zcoInw22vR3t8tf?ck)4nu7~uyBfrJk4z#eBd5Uep$O`V2uhO{Q#jc%GN^lqP z^?mPw{sD;X#%Wh*tQ$TgA>a92W6kbKa)$qDRAzS|idAAKIv~^aYIE4-E_%Wsl}&&8 ztQ#C{aN328>?#qM)m<6Z$n64o_ z{d{e~-cOBF!$IR@h0BVXkT%Z0Nq1`k67Jg%dSqHTmE2#VKLcTZJs95Dt(qSDaRG*+ zZ5TbR&HcmXy7$EhRtr(znqW0k5BAG>>@LpqWy_|Ra)E|dEy}pc_atbD-a{m3LIb~^h zwvA@;_ll9TvM>dR;r=Re^tw4KRA?npApt)8J}a3$sA_D-LgWQ1x^%I=AHEG{&tN|^ z`>?CjP}(Q+vX1y;j(jsSIu@e4LU#Fq&!!1l;p+b&+6&*4k`RTkTIgvN)mS)bC+5OE z<>j7X0SgFlN2=j$$fJ(f&GsM<)XjI}JerfZbhY?brP@Ws72oIYh{7z)R+x@pBcaIp zNU~6pF|mN%Enn0tGPb*r^mlyiCv$_AhL{s^aRNDLqonxD2ABGqVwVwNG`NKaDxY4M$upqhwWe0)}5&K7& zB=}yF2w{lMxgE$^&jJann9L+uts6=(R<_xTmisZ>Gs9V=lK#$ff%(m!2}yZkqAX>f zjOIy^<^yA}FbvWzNpLS(`=cuEtj7Ozd*ycXGK25z5VIcHcl}7qz|BKm?$o8-dB@Fw z?a6{do}K3%1s@}lYZB-9a?;zASln zh<^HzJ@(Fk&_-TAPhmGehV|R6Fs?Y@gNHMedxh`sg_|GVx78w_Y zWuc&~rWvBj?W06`srv2_^YFfpJrm}udtEcdA0Qk&BSOJ-?vvuS3D%zk@%matlqKid ze3Q{yldycly8Blxach0MQs4DTDfcX<-L%o>dBW;wjdQNFJ|>`=iJ?z<%D)SuRjI;O@b zO5W8JkMcJ@IlHNnbgD)agdxQSyB(V4 zE_(g=9q-ncq>krW#0^UwP3Cz08ofFbEta7({`zgWBJ8@(h8&wZfiFx)Ya6YCoNBpl z2j;_$VMF=gcxL*@ZO)M!tKRREJ>GC=$~gOrL-pM;r?2t0lEN3lgGEp&9Cem*&Q_C& zxGrqI894Yk-zxF8GO3GWMHec!r#Kyc|1Q46wzJT4ZX1*Coy9pqN?xCrn^FO&9|_kw}%8r8B#6|E?} ztu!M%B>$(NagBX>lDDj<8!CgnM#cDbjmV3U=v6d&e!VFDH`)Tol5Axe@EB$LUi}dv=qsQL8hNK(8`Fv2Cs~k(KBe@t9H*1mOLcYrjm%1xSdZ^t zd_(sN3ymY=bk;4QM+}B0-F!~}>YPjC5KL*&52{BLVk$)v zhWK=RK0(a3#P?@%tHDwkpe7>bVHJMEl1CeqImJ>amIgrc1w_myj7=|Jt^Erh+0trD zt)-M9#YXeaBUK264gLcBiJeI1{Z~r6h1KNq^4F)vI96lgrn~%%f0V*KM$|Jgg2qXA zUrGDtEjMi*ahB`0U^usv>J;?h_gq^e(=A_$!Lj2~DV_~G^z)4^ES+6)ELz5C}TwN;54(kUn zyt~MW0%ERl7+g{N#0+2@rZ6yjXj#FJoyUI01mc4=#wSh4YqMHqR-VnxS}m=fJgWik zPGW_b?~_4AdwA(I%yI<|UAIft1Na z4-1RFRad^W?2!N`>A8hOIg`uR;JJ1vk%Tb5?Z!;x-z!*(BS;Eq;?~YS`;wut9_f_HC$QbhKVM%FZ=lbFLoSL0Gl8r)D$=KR< zMC04V1D!SLCEP^g7B(-&X8;!6y^kPB{NV~r@307Dsq&*%p(?`1Yv1<|+Ij9uB^ z@y^>S32yTwg}ur9^3(NpOb#ZPU3OfvIibq`v!MqQhj}&z0qh>J%&MwT!vHq4);FcW z`75qkvYI-jISGMVM+C03T2JC~Xr$(;JIrS?cc?x9S}lGz+PF=hUFk?b@& zQgH)Q@`Mc!er@HDA}zuT`wnsf@gGq3%|Pe$5VFhFa*-4HtqkP0iTVRCqXB8~hi1FJ z(wgSy;1h)s$mHIo*gKF!XpI|cq!9~$TWGLPhb-Tql0?1-D;jSXMMog4VT2;K$dy?7he8_#MkZ0u76{K z(6EaXa9k#K&cbI!?|tFWP&#x7p1e<- zH)F$i79RxWQ;RwdVMMzA&Pjq!hwMf$C{H3Mw{Uk2vh-s*nFV#*r>v6Yog4T#4@D(f z!qgPhl15&s%thxx(|Ud^3fgj)+)Jyf49obuS;`4^ZN0@<7HjehG;#Wq77}SN{A|#~ z@O-5iryZd4ks3S<-4eowKZiDl@^s@F2Ib5&%j-QAQKK)A1BB>XL0uwX8Ug(p$* z)g#cU6myLv%4xq6gaj1zXI$_-q<}L0P;uY-M&I$dakax@n>2ir ze>A=!0TEhXhf|VWB-P6X+>e7?C5e2$NaS6bLp(G}*?@(j*>PUSUGi^5!!?e4#+nrs zYXXC}U=F)nrZ-Y==J@KsK^_QOJb829AqyLU{jgh*wg<4)ZZOh3D9(5@A^_fQ*p(8> z&rdSo#%P|l*=I03zW^c1@XNCAxp9^hV5O=B{bckzr)D$%5b*MS%P!5bUNXbKj8ls6 zk~b@%7@(s=2WLc?{osVb-txnI>&qA~G`268F?>Ic0)k)~;gT5bo5x{{YpcGg4>|$g z6|fYZ7SzmGX!_k_I=lH=qPjb$3;wACe%bRWb$re|EvDGzhy=#jpnGUVZXLm686tZV zUy_5z@QKZ{p!;}MWufHe=A%8EYTbmCf;SKj!jV+Org_)4@chHjWq}Z35SlRx5dCYO zQ+fPVtg34SJ@p>Ud7K9~od#+A_K{6mc2uy~Gn)gHd0(}U#ZtnZC8-iGTt z_ExeMp3_^Ic}NI~RlFuECP&jg@UY!iOc-RSS<9xc4CFYxC(j5&H;Y9W=zNBy&LhG& z!~(5k(Ar`6rn)Puo7(@d_*yW|`^Q~;#E6daFga3G-57xHN`KbW3u}E(el};Fci}f~ z^mk3ry8pt*0xu{^mqLUt+hM4uNgH6#0EDDTUlV+rFq&uN$Ad%U(06evn?b~IvxV>qRQ z*t~)O1B93LAbh%CZJ(`$dEn-q7HZE;6StSdA`|<_b|lSv@b)R7ueq{Wly~rblxi`5 zFj?34igzK@;rFyVc@18&z+n40K~wmTqr0W*nQ%?L?GrrU8G}uSKXqdE-KPUF2hE3G z7a2eqPa@7e^S$x!@4Y_~YsN1cz@qGkW|bF*%yXafjN5*H=}xZ5v^!(Uw*8xh3;!^cqe>W%kU`Rw|&dyI2ilx~#HIPc$A51ZGoKJX7p36~+ZDrOuhSt6rgFd%b$mF#F>i=1a_n%e#Ib$&_OIsAQwk2562lBBo?`|q$KC$ zdSB*{YGLd=4{=t;zv>=LfcSi80fA9ho3W>jZQE6RIr$$qdb4PIn$WKqa-fn_lELXJ z#_NYlmqW!*edL6too3Cu;_5_oVLck84I$`vtwv9xIS9h?78XmbbSrdQ`b}i)wA$i$ zh$_X3MKW=O?Qyifn&&V{Rhn(@1*OAE#+tjH?(qPwx-U(KkACzj-0a$*$X+>1Yw``G#F+*~uk0{@HIWw)a`sW15( z@j;2VR0kjc<{g1-5l$h#0(43)Y>wTW$| z^+_yb3tG6=>9oeoP$c`m5s~#>+J7qO`RlFHA+DWdDsJO5k{;|8{IL7= z8jBo4jrA6z*I*t2v;7i06drqP}q(fs?7o6APn|y1+MkIqI|i?ke^UJwlWt=O$P% zI48|99~ExT9Rm$x$%F+9nyZ1Tzncq$_=({y`5E!YcZG@>#kU{2zgIC>V7sq9{Kyw3 zMj7p~^Gj)HV=2b~^iu{!eQ`~$o2Xc#6B-6)pM0_ip9;16^LCSDQRi(b9kZ~!=D!da#LOJ1 zglHwlxkZmr{bMfN{ExKnKIr&rjxqgh7vG=`-w-3LIK2~7+tC>jnU@!s;)LFiJE!_3 zfBw%tIop&`&*h8hymxe095&K-m7BjN`q@-J8j@RmgQ8(;eG_MlBJ0#&ECP| zaL;G(`QGqGsDc`&>JFSmR(Db+&52z|i2~Vt9%}9Lof`n4?O=r1!X4|pQnmdJ?4CQK z%NYHm4ETD?Z)s9U?$hf%L>NRrUMfbDc|m6sh6}gn%6KL_C5^vxHF90_6CYstK_tVO zYff1ekD*Q*uNhz~W`g3Ks^&idFBNhoJBvtgKs~=S!g$3kg;;P;a#hwfdgCT&P)>48 zka4A*cQ6cCtm0Zwb%k4Z7!cKX?s0((5PuUF4)zJ`?pDJxjoSCC%j6Z>Ci{VR1{`Cx zY|JR1kr#d@YnUad{uaXpOh8XYstvgXy`u{1JoZ$eXZb6ngYkBZ0Ze>C>h(Q$l-c9KEnZ|lvyTHHwNy*T2V z2nxFlwC=P-l=2bu)Ct=Uf|*R8d?%C#c%UU}z(SZJsO<@R<{kzCMJ$#b@zODUdLr5< zUBy)ckm5WW^&F_}VjKF355r?SJNA^tuc=4uebqX$GFgP7C%6(NCMD0FY$BI1B)OBF zGnLS$;=>`702?%vn!REef3GO*Tm_V^k&gH(k<19AG#phX&rCff$7;5+z2i;mhY#t9 zy06c7a3TRQ0R5$y1CrD8y0e%c%+urdIs5G%il)i}DBHXBS+KR{qt%}&X9&FL)X`>q zQfHVI2GLa^mZ#%qNhnC4%n!_5-n-vK7s>J~Boc@^_=a8C*=a^0OqSvaUcLF+`tD*8 zgwXke5Wu#?Mq`KXA~!{Zk{Y`S`z&g{mSJ^XDMG$Tj?yCDkH@@Jj8PYJIbGVUi6U;= zn+3L{5g6HA46!JCp^lR#wH<3Q|p*sm(9r8OggpeUWQU8jbU0xKem5cFK` z?4k=8Ba5bwtT}JbMxg5#l|j+igZ4tu&o7R2Fs}H%KSo&Nn62@+xTn+VzVgFJCUHj( z3(?xCj`0MY@KPCJRE=|A7~G3aY87s|Cs|6dir{I zQ~g_pVjvf+!TQk-V)tX1w+?KUT4T8**n-*31yE1A5*i-oEIaOt_s?S+ra16nX-`pS zb>xo)20#vo)AlGquhI9+gN^c#jVrp}mPZnfuSZX-T8IzWkpXj?U76apZ!1f#k)YQQ z=Fh2$O-*NSc*eZT_fI#oAp{IC!A{=e*o0Tt4c$jinIQaRGOH|I5nsEO#MV*(XBX{B)lt(_>$=Z%?btCP;!V z(a;w&Sq<*tQszUfMU|?@Mq871V;qIbK9;d8Rj!8Mahz1L3h6>dyic+^*w>}sZeGkr z+0_Z}L+l!Y!8Gp6EP*Ejmr|Y@L%d&`l~CuUx`c%{xTLJ2SMIPeF@IX9?EDFdBa!qf z+;TCh^&^KB!@HeA&A zps(SMrif|T6JD6zFO=&GIgSF&sn3d>v8~+&pLZwO-Vc2cXa;kfSQ7cK%Fw)nS6~Pi z*;|ZZY%39`uq7y~Y6`>Z&a#R~!U6DDV$PqFy20kr^n(g7Nri;7f9!9dG`l)03ZvB# z8ceqs9<52*_@1v-D6=){Fch}VZr#EVqNjYjh=3($9-l?&=7*ogvrPStuOv_Dq8xhf zm~F7g^ydh@nzF=CDwDBczD&_3Me&t`>i^}xW0w_)jl!NXu>D+bPN%8z9HA?hQQU6v)r=1#b~Wo$2u;G}l#uRgM?aIiW=8%e#mf ziw?0|o>=Gqz>r;)usO5Av&6F_5KDFPNJ(=HVEXFl=cdJPrHN*vF~TdC2I@_u46;;j zf-P$cfkgedu0PgvYzA|vy3LshC&iQIH&t+A@00}?xO4$Pix{%kb*1-CdziDMC8fR2 zlK=PGFlQW|(1uMWBe!=xuiiMh5cWY?W5>tGBo*f*J334HU|W>WtFB$wjn;7M%;gaG z;7^vrow|5GF=0M|x|t^UWg?l&k|`XmDcB5#NClcQ-lyxM0zYRP6LZG$-@^da^TF(G zyJc)ECl+L6`Xg~8E{!!Ai ze`50=VmoRkRS>De+MvLV8*@WWMLLmV9SN?bf(prsJ=uhJYtF|ybB0=j89#y6JUF-O zWP7MJ$Ys_bOWp~Tf!~}}YGHUZK`31Y{gTFXJ=-(DaB&!Zjuk?6T!!pPSX zy+2hbc>E4`1I6D9W-W0lcJBIdsn1)5meuoq-^2{k{>>Vp2MfS@?k{J<(GDlxoOlbZ zd9U>qzeA73PlEsWwD2v|{m6LOeIg&fd1|w2>UHI#FAY#k#4GMgNF(6>Td?mPW0k=hoa5fyWmm->fAE1GZm; zSyw~3fIcJ476MDO>g&--hZ9*H3!Da39{kMFQ-VRj5Sy{RcD@~^_IqNv`1{m}F0fbO zp`!YDho9lszY%KB?^pcOX@8NT9h@(}I>XSkafpDJId@mV1DlP=p&W5-FAjiR%*!>QL$Kce zuHmA6g~Ur9kO~aK(e6=z<|Yz6O=^5All@9f?T-NLU5M-Y|GE?!MasAKH)x>KT~qA+ zGh{dqvtg|;ijNG_5L;Ho0zcVV8H^Wog}r3V_v|2&C{7a6!+Vx}I*^6t=*wBPmXE^x zebA5&jM21?X^EU+?O8#4HvRgngXN&0))F}4x@OF_sFoHR1pJ)c8yiYjOOb<*Z z60P+S*83#KJCYcm-PoEnTk~(ZL3cwM`)XcjG8&!|uM^dtk#YvtKq?^!br+!kFu6@G z;mRQ0kt+KH7LfRH^XSw=0{62}OIeNkAPxYO!fY#BN z}j-TTFdBobjy1&RYh102#YDgL#urh%uxjkpbl7F@Vyd&nYx?x$dwi zpM2zn{T4a~sP)~MSR6T)i$5us?hyK>T24NpeF+x?_np`x4Vyo#e@9X~z+(53!hF}7 z;oTKQSeW2#@OK|4b8U5?c5QOTnDCqU@;)Qs#Olb+RU1N89V^Vb3gu7Y5rv+V-)Y6E zx7`BUe$IA3rd%AHy3EEs-VINvZxcOj5V`P;(yZ`=U$C22CE3?Lq(IFbX=dPZW$E@> znG;Ki-RwETX%r*k@<(|-YH!{mR`o(9k$Rz=MfBo9#47490Jazp+ zIaa4rxY=J52-wY75NnKRG0HP7yD?YJZTwvNJzOYl_ljg5eZ?SmGqJ#7 z-5o?kV*Q=_QGe93vIX1+Dq;wS!P%F_X%)|vNz;I$Aj`;Xc;Zg??k2;H=L(+aw#z&Z z{=^`ZDR(8_;!4N_ZwQu)dE{|h)mb6#>JQG9p*F>T5778{G$|Fe7)tn+j@V-pU31!6 z_Y#L!U*6zLPG{vJb)=l9h0wFDvPTPwK5^r)6v5-_kYOcH-g0_1#)Cd0Uzwnd8&AER ziXjvmtitcVI&93?lC7#^==f?4^lBrYy<^szV1yWl<>U5JK7%QA~_Gh47Isg~9eL#IEp_EN+8n&l5m(*2AlF&NnSN z{Oo3QqN|Pk)#biRP}(%d`y(|8j2d~$wP@9nK~SX$%j4<{VJ3GThGP}8+Q^^_?AC9$ zVw!qeb-kX&e7~=B(GUyhO%T!;gSlxgVsEz4Qeua+*Nm) z%)W_Zu@@A1WUBhSNp4yncafPM=3>2nDdn;O$1(O5(d*X*&KB(YcSQjB~@02l$^3F zNi5^*$a7mn!HKo#2zb}&vNj;X@Nq&|ovWPIn;Ga-f`twO=$@L=(Jada(zB1>U zrDlt39cf~yh7gIx;g@C*?S=^n7@=PYc zBI=47O3G>C`M{n*A8b0(_rf@RlZI`DjYkqweg4%czauRy-Ya4i`*mSrb;Pi*gO^6Ua$$0m8!2Ql1 zXM#?97@C3q3|_+%P$}EPUlr0NDryMq3gc6HW3k0omwk7-G!~U#Dhw)?U1u|HR*T`3 zKL$lAd1LQSlK*S-6MBJ{LQVKJg)O%biKvMGqB#~3(HXYyp^7edpTZyFDy=6P{u8In z@L5(s{AdcAkb{_(GqF-0Z~0#hHBQZlm5NzX8UI=S{kl?8M8&*|IlCmzjyP0B*MS(2 z?>R4|f?{&1d|X?GOk*tL4@6)yv7cIk6?aKr#x+P8%|5PwOUVkbkDEKqwH-;`m4LtaCu%!oE zcskiXVeF0%Ht@`>@a3?TZ5v(cPuI$@HYRtr84BqTn|^+|x@Zzeu}kj%ytd~Yxv44T z^M`0F^mhS`EnBZh)kWu|gF%c;P9)_wxc`--O@srh8RZDmBVgt8>Gd)4*WifJZ@{OV z!tys#P4ADS{^J#;NkoF-&bwB;_w)jB3{^QSzhY?=Ip7(?DXdTz<4P!eZ~rG9Lq-WH z)1Mt!%ZgAxjECn1_B}q5XAr`oTiHrG0Ff#stVe}j2wdC-Z2!OI>OGR6Ve;wJT$sW(vG|0&l5Ge4Dk)7V$k>M16ein`2P~(z#E}F{ zW}PT)9sTQh43wt`Wg<9QUtx)Wz&QDD>AnQu5vJ=S7GoM0CcjcCxKc^cED;x;^RLPh zFw~if;OP%LM?U`ZLFQM(7Dt&MfRbJp3Te=*ux2OEjS+Qt3uREmmS^x{pe>s_05p~7 za8z|LI$D!jl~*h?t87dE6m-(CTgXUvsa(EM9q5)qfp&_O(LAQ2t+>dVM}t=7>#hVc zB*Xr0WV3AgCp7?cHzq!N9A8(H`F|G|3t+Z@t$N=&Aadtx+o^Ui+`K2x)+%O%&%IL~{YSP&`^@_y}rU}5VLZL9{ku0vMb!kX@C9u1UqJ1pjC3 zkWm1(^ybX=US-|~?)0T2DN2@xd|%PlT}(h}SF2~$i=q1mL?>?N=^B;rGd(a&Zi!aJ zn_=L%9U_wXpYV@>m+DP#O-TS(q4W_$s700r-AJ0KL(!$Yp!^zZ3{%Ja{=b1LR@Ea= z5v$VbppD9|M~BuYPKJNNKp0ab0!K46JUazJL{PdjpO*@v4iCtg%8N8>_17!Xp(UL>_(wL7BQxUY)6sbA-e|@JZjmeZl-{F)nXUm9idRT0Ya z%#8Q z_bGq7(OvI6Sna%#GRh(Nb{z9BMxprK2wFz%!UY^J9&t{*&H}cV;1ED&*BDMe#g(yG z*ipm#e|^N-jtCFgVS~vW-sil3g*o;JGRT0$H6pVxYT*;)zq!|@$#NSN^(dA+k1m~Uh4h{^iG zO~Q^kHvemv$gGk98y?>zV_l%(;dNU<)%3RmfON;~OcjJcrByliPyFd7mh5NzbJp-+ ziLt>X6C|7n6a#uaC$G`D{2_MuhMDq3mmbFxAIE1Qn$YHVgvxRQaSjLJNUtKWp%!^* zlr-UZ5w|CIJ!2RtCyK^bqt7SAz`0; zX?iTC=1^Di;Im|ld*e5!rL+yL3=c#*vqAy)Z%!HTC)P0CoQ&5K|NSqFz(S$4xou7K zb@REL$r**28L*$>FZXrNf{Ef?ZPBDO=Xu-_T4$8_X5RJRxg$i9#Y~GFt@CrX_1Y+g zyKsT{=lMH#7>;;0yao1-YUrB-be@2E$uoSWLpeW6sSN;#x8paPoVy>+_gcvjuvW~3~E=88QrDTmJ5-wK=B~68BT+0w+30XtNmd2FK zps@|wEQu^beB>quSq5`YbEf(y?yu*(=Y7xfyx-^hJkR?%=kv|`+iI^R2RK!ay$F_y z#GDfc`2k*1j(YCsB_6{5SMZhV-C8&|dLxsjvzrzV4``T))62_VSgu?E( zL0i)-{y8Q9E;ldPM7%}%j9?oILd^_`F1xo;`M{PK(&wOFko|RDH_4^UZ&pI}LdTBX z-m%!Rs2JINY*Yf!@jO+-?*q%rSK8;XUDka8e~7SYH)uY_U9Uz&w!Hf;dGlfFp8l zrEP*vT@3c)0?o7v8yodgC#HB!FqxU=#S=?nCiPDaK|8+>FuEjI2g`GY0lw}1Ma;E; zW16p9h!vS_vaK_7_^MvcizU0_K@e#ww)r-Xhk;(prRoBwMUsEnhkG?7xu^rK_g6zY zYM4+Fz1om5q;*y`@gJa2?F*0cMcUiPqra0jfYeMkFv?-(yN7-3 z5#FS?(Q|9~f*lr=fZiHJ(){*-Z2VSE*-?%|GK98QV8PFFT+pDFEyJdqj@q7Kh<|wL zzbSW+o8ZOGoZQVN@4jDSy6!<+df;Xh+^*k;y$co$F%OaKrj)5#bT>J`g|=a~OxA`k z*zBScYDMw|1{XYY6g)CwbTQA2!V)Nkl)dRlEIU3#niTGQt@%`az>O-_i#2RBrf8@! zS+sTB`8V->(J?mS1aBke415NFh}xB1;3`T)15C zkwtupXO-SA9y#VX?90dR(|tby{D?;v^L};jRIr3xU33`f)`qVdJLKqVi0o;U{72mS zRYF4$#jV_p`UWa7j#pGNK}Y<8nMZ<_{c z%RIwLiF$Y;DP2;K(z?iP1bOZ3z3vY$zDTbNmKu0}_nA7{KS2TUvI72%3u3CEmUrnf zTKJ*iHHp$w?Ztk(UdP$2Ba}p|v32!)P<3zdcr&h6S?Qa7E2p7 z;>meP7Cufq%XU;ZYw&u+E%}QPA)#Q_l}8uGVeW^##zM`e0{pWMvcS`=qap!8n*_3f zl?F^unnh!8<#PYeA==>;XA7Ujv4Z%9Oh{dzZYIs(x4vtYw2r|aL-Yb3E=y~zST?ph zHEC^=m9pz1^60JgiDY}b4gEpA{ZOlxvfMXwMx{x68@s1{-DK1uyJx7eeY?uS6nT2K zAcHMqE^qNKYQ_^@jqPvjEVfQQLsp+*@Toe{Y-Nj6aTusE2~<>kW0CBQ0*{+yj^k8& ziww=cvzlc0JP0XJPJrA9kqJZq|KH_!W8ol6I2Lu?=eU?s9@ql`)|U1b1aptu{{ihg BaNPg^ literal 0 HcmV?d00001 diff --git a/nvidia/heterogeneous-distributed-inference-rdma/assets/test_nccl.py b/nvidia/heterogeneous-distributed-inference-rdma/assets/test_nccl.py new file mode 100644 index 0000000..469188a --- /dev/null +++ b/nvidia/heterogeneous-distributed-inference-rdma/assets/test_nccl.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +""" +NCCL Communication Test Script + +Tests NCCL (NVIDIA Collective Communications Library) communication over RDMA +between two nodes in a distributed setup. + +Usage: + On Node 0 (head): python test_nccl.py --rank 0 + On Node 1 (worker): python test_nccl.py --rank 1 + +Requirements: + - PyTorch with CUDA support + - NCCL backend available + - RDMA network configured between nodes +""" + +import os +import torch +import torch.distributed as dist +import argparse + + +def test_nccl_communication(): + parser = argparse.ArgumentParser(description='Test NCCL communication over RDMA') + parser.add_argument('--rank', type=int, required=True, + help='Rank of this process (0 for head, 1 for worker)') + parser.add_argument('--world_size', type=int, default=2, + help='Total number of processes') + parser.add_argument('--master_addr', type=str, default='192.168.200.1', + help='IP address of the head node') + parser.add_argument('--master_port', type=str, default='29500', + help='Port for distributed communication') + parser.add_argument('--interface', type=str, default='enp1s0f0np0', + help='Network interface for NCCL socket') + args = parser.parse_args() + + # Set environment variables for distributed communication + os.environ['RANK'] = str(args.rank) + os.environ['WORLD_SIZE'] = str(args.world_size) + os.environ['MASTER_ADDR'] = args.master_addr + os.environ['MASTER_PORT'] = args.master_port + os.environ['NCCL_SOCKET_IFNAME'] = args.interface + + print(f"=" * 60) + print(f"NCCL Communication Test") + print(f"=" * 60) + print(f"Rank: {args.rank}") + print(f"World Size: {args.world_size}") + print(f"Master: {args.master_addr}:{args.master_port}") + print(f"Interface: {args.interface}") + print(f"=" * 60) + + print(f"\n[Rank {args.rank}] Initializing process group...") + + # Initialize the process group with NCCL backend + dist.init_process_group( + backend='nccl', + rank=args.rank, + world_size=args.world_size + ) + + print(f"[Rank {args.rank}] Process group initialized successfully!") + print(f"[Rank {args.rank}] Distributed rank: {dist.get_rank()}/{dist.get_world_size()}") + + # Create a tensor on GPU + device = torch.device('cuda:0') + tensor = torch.ones(10, device=device) * (args.rank + 1) + + print(f"\n[Rank {args.rank}] Before all_reduce: {tensor.tolist()}") + + # Perform all-reduce operation (sum across all ranks) + dist.all_reduce(tensor, op=dist.ReduceOp.SUM) + + print(f"[Rank {args.rank}] After all_reduce: {tensor.tolist()}") + + # Calculate expected result + expected = sum(range(1, args.world_size + 1)) + expected_tensor = torch.ones(10) * expected + print(f"[Rank {args.rank}] Expected result: {expected_tensor.tolist()}") + + # Verify result + if torch.allclose(tensor.cpu(), expected_tensor): + print(f"\n[Rank {args.rank}] ✓ All-reduce test PASSED!") + else: + print(f"\n[Rank {args.rank}] ✗ All-reduce test FAILED!") + + # Cleanup + dist.destroy_process_group() + + print(f"[Rank {args.rank}] Test completed successfully!") + print(f"=" * 60) + + +if __name__ == "__main__": + test_nccl_communication() From b254e6f6328aa05e9f0a4f5e94268dd47bb45655 Mon Sep 17 00:00:00 2001 From: Csaba Kecskemeti Date: Fri, 23 Jan 2026 18:59:01 -0800 Subject: [PATCH 2/6] playbook rev2 --- .../DISTRIBUTED-INFERENCE.md | 14 +++++++++++--- .../README.md | 12 +++++------- .../assets/devquasar-logo.png | Bin 36156 -> 0 bytes 3 files changed, 16 insertions(+), 10 deletions(-) delete mode 100644 nvidia/heterogeneous-distributed-inference-rdma/assets/devquasar-logo.png diff --git a/nvidia/heterogeneous-distributed-inference-rdma/DISTRIBUTED-INFERENCE.md b/nvidia/heterogeneous-distributed-inference-rdma/DISTRIBUTED-INFERENCE.md index 4a68729..aaab6f7 100644 --- a/nvidia/heterogeneous-distributed-inference-rdma/DISTRIBUTED-INFERENCE.md +++ b/nvidia/heterogeneous-distributed-inference-rdma/DISTRIBUTED-INFERENCE.md @@ -384,7 +384,17 @@ python -m vllm.entrypoints.openai.api_server \ --- -## Step 8. Run Production Model (72B) +## Step 8. Run Large Model (72B) + +This step demonstrates the real power of distributed inference: running a model that **exceeds the memory capacity of any single GPU**. + +| Component | Available VRAM | Sufficient for 72B? | +|-----------|---------------|---------------------| +| DGX Spark | 128 GB | No (~136GB needed) | +| RTX 6000 Pro | 96 GB | No (~136GB needed) | +| **Combined Cluster** | **224 GB** | **Yes** | + +The Qwen2.5-72B-Instruct model requires ~136GB in BF16 precision - impossible to run on either GPU alone. This is where our RDMA cluster shines, aggregating memory across both systems. Memory-optimized configuration for 136GB model: @@ -524,5 +534,3 @@ This playbook was contributed by **Csaba Kecskemeti** | [DevQuasar](https://devq For a detailed walkthrough and additional context, see the original article: [Distributed Inference Cluster: DGX Spark + RTX 6000 Pro](https://devquasar.com/ai/edge-ai/distributed-inference-cluster-dgx-spark-rtx-6000-pro/) - -![DevQuasar](assets/devquasar-logo.png) diff --git a/nvidia/heterogeneous-distributed-inference-rdma/README.md b/nvidia/heterogeneous-distributed-inference-rdma/README.md index 05d544e..ea86d4c 100644 --- a/nvidia/heterogeneous-distributed-inference-rdma/README.md +++ b/nvidia/heterogeneous-distributed-inference-rdma/README.md @@ -27,8 +27,8 @@ GPU memory → PCIe → NIC (mlx5) → wire → NIC → PCIe → GPU memory **Key properties:** - **No CPU copies:** Data bypasses system memory - **No kernel networking stack:** Direct hardware-to-hardware communication -- **Ultra-low latency:** ~750 nanoseconds end-to-end -- **High message rate:** Up to 200M messages/second +- **Ultra-low latency:** Microsecond-level communication +- **High throughput:** 93+ Gbps validated over 100 Gbps link ## What you'll accomplish @@ -135,7 +135,7 @@ Both planes use the same 100 Gbps ConnectX network in this configuration. 1. Install the ConnectX card in a PCIe Gen3/4 x16 slot (CPU-direct, not via chipset) -2. **Cooling Requirements:** ConnectX-5 100GbE cards generate significant heat under load. Ensure adequate case airflow and monitor temperatures with `sensors | grep mlx` +2. **Cooling Requirements:** ConnectX-5/7 100GbE cards are primarily designed for server environments with active cooling. In a workstation, ensure adequate case airflow directed at the card, and consider adding a PCIe slot fan for sustained high-bandwidth workloads. 3. **BIOS settings:** ``` @@ -416,7 +416,7 @@ sudo ip link set enp1s0f0np0 up sudo ip link set enp1s0f0np0 mtu 9000 ``` -**Option 2: Permanent Configuration (Production)** +**Option 2: Permanent Configuration** First, identify your active internet interface on both systems: @@ -549,7 +549,7 @@ Example successful output: - Link type: Ethernet confirms RoCE v2 is working **Performance expectations:** -- **>90 Gbps:** Excellent - Ready for production AI workloads +- **>90 Gbps:** Excellent - Ready for distributed AI workloads - **80-90 Gbps:** Good - Sufficient for most multi-node training - **<80 Gbps:** Check MTU (should be 9000), cable quality, or PCIe slot @@ -643,5 +643,3 @@ This playbook was contributed by **Csaba Kecskemeti** | [DevQuasar](https://devq For a detailed walkthrough and additional context, see the original article: [Distributed Inference Cluster: DGX Spark + RTX 6000 Pro](https://devquasar.com/ai/edge-ai/distributed-inference-cluster-dgx-spark-rtx-6000-pro/) - -![DevQuasar](assets/devquasar-logo.png) diff --git a/nvidia/heterogeneous-distributed-inference-rdma/assets/devquasar-logo.png b/nvidia/heterogeneous-distributed-inference-rdma/assets/devquasar-logo.png deleted file mode 100644 index e90b1efd3d7539398171af5129a7f9072b5a7a4a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36156 zcmeFY)k9oMusys72=4A0T!Op11q%ds5AGf&!QI`12LizYBm`%0cXtcUK!VSYlXLF9 zZ{J_=z0Ax*Pj}T?tGcRt*N)OqQ@}taMFju=LrGCq3jh$Ge}V##5uU%=OCuct01PO} zN@@Epo#ff(rtZ#0)MhyybNo;_bILGzAqP5?>s<>-pod)9L`6Z;GusZM7~UNlTO#3W z`r|Ou{`|9+z8YqIG;(h)CFOIPWxHn&4|51Ux30}mLaW3|3DrcUd-IhaZ z>LBr&4V6x4g&A50gTgD#e~0NEc3Vh?cHFUAGYH&kBb4xj&M)|kTobzSNd1)mmj?@) zPC9K$Yqj|Y3Jd<`sTo~b>bxh>@qEizSawGx?_>=AUu$Uz-80YGL%#3T=8oTTj0SBQ ztMBvXTlla;f-^aD4vUGH|57Ww2#=Dy(RU8q}?QF4R1ojISOGOS!?zLH}nm54)oOnanZPDq*rs92e75iNBe*6r;C~YvATpM{K!Dj_NVNBPTJxJSaLS@ zgfegQw5uB`$VswLv_{Z3?J#^cPJf8z{VDz5IL0|XFdehB#CAlZhb%%#;`l2)MY$1O zI#|bF;YOMnohBx-tYQOkwS{)dx0o)tCTf_fy&Hzq&eriLo8_K+^QZoZF|7&T|aii^>W|1htLriXFut@Y{I=y73 zOyJl!Ewq7ACj`-RpLP9Lgzj1ehfSnI!F3u8 ztc+dHjMAFX_{igfyokH1j+F*(u<&duOCMnCm;frC4QqMD=D*RhV?}-cjxUvHf^E+d zff#R-Y3>mk(~{!7jt#QU9YZ@-Te!h`(-lO>jb(kgZZNJt_G;$81-X;EgK$)LaUptw zY8^bBLyToWj3sP+ezHOzJ(w zgz4P^w)OM;DA;AmyDKUR{?J$7maQHlDt83ZFENY_9RQ6?&SgQJ|NUPGbF>}DtCGqv zy4}mqad6bCneX@)>>lQCV9SOr#ccWfp)9Ku9;FEXqOXmRa1=o1G+;YDsnvdun{S&i zsym-CIdw(imejw~qUNxl0;{ugyzU{H>u7^A?33(_6F$k}ik`ZeLq8>1Gt z{`~Q&kw4HMH*ZGhFWN7GN}VWljyW>jJnJ-Nmq_W`Rf-7kZ>rKQL#>G zZl!RmMi*|dK=*eX%}YR(zEdAU{D-vq1xO0#)vYT=VRi=FF6;v#dhA^uZ#Xlu;$K4G zsP*jzB8gAgQySP`fWoo{U#@=vRd0HIW{)a907(L3|7RuJRV)pF42ND+SGrAj>r5rd z_o`uR>J>K$B_DuXSoab9Klp?ImKMkW+<_GrGSdip!Sj537@yf5Y108}uy#1>|E^3D zx$2YGr%zb$Y~Wo5r6gai-W3e#KxLV@W0_8w{>8tpfHu(l`|ts64_Ju3#l^vU19-i@ z9BqR;ISTnCf^8o%u1sw&-2c$&Sf9?~LIgmbTQ6h!Uv0hkeY>SrwrtU`w*l zRhFyK{tNnsOEzb3_ui}{MDIQ7bk(^d4R#+G_N7{~_a%m6H5VLGKyp_s}I3zGkB!o3W&bnLgV|M)(Mu z_$WoV+|Tp^^p`?K;3lgKy*~4&cn()ObT#9=vEKnmYkMd3B+*%!4hIIlPs^Ek@B+k9Z1HtR)L!5A?&+fx~hjM5v0zk5a^aL!m6%g}#WD-6Lw@C27H zo-GEwz5Kp=6KvOYm>!I$ft~nQkO<&ldq}6?X(fkkZ{4id_lRrYEo2jx!1&Bx)qniO zQ^hxrQSzND{Nw|)VO>4B-|f+P#63%9d?tmk!xC$|6b z_a*Q!#@v_y&A70J`$(ed{>Y)LUjsu+9BOs5g;w?>r@`a>!^kr#WQXYL)K)b4s*)zv zpx&crN8u-M;9@_u3&;G2M@OUtfJ~jS>E<_34y~;6X%*&xN#rUoBaAJj<^N*)kJ%^M zr)TNnBlZy>cqW!Jd7n^jU&G2>O{90*E{UyM4eI|3ch|(&P&hF48g=2K7tB|`=3b!qr|GzMRYq{Y&D{{^bZ-dx~FyAYR_w?Ydi~-=>Jp_)xN{{z=qhL zcDDAD58TwItV7t(KutCi6IvwJjGbEP5_NXa{zSOaLQpF z^pAK(6rlf`FtQ8<2>{K+@H_{lC(#@*cZoXd_ITOYq5`If``_RV?5V_Ga7_hd>~ZH` z#rTCk#;=(Z)b<_1)OX1Z-+i}nix-al4?C&>D+Hl$NeJ9nPQI=bX-&cgSX2YX$z7S;;np zN(CfDp3F7gCiX0APvI#MHIPOFmRY8g12EeL$!-aDGDYF!v2QGVLATJ)ZNr&XdK_=q z_;>k8JS_mhUePw3JJN(VmWNgSVTbc$lpEr6Zh>fX)DNDU3_EB;9v$wL!F_Y-*%P5% zfeC3N7O1B6JRcm0{xZNA2_R2pI>(w(uU4vOGtOh15yveM?$`c-!pS)4!&mHGBRs&o zh&NpU4X2Wc#OcF+_Uc_82A%>ZH)ZC(nZPX<4UC^X6_b>!W_u~xULSuIZwM?jBYsVv z$K%X5-k%vE7V4oS%cy_%ATZ4AXB@GCxBlh%ukrPOVlhFGP|Ev8x?x?f zsEFxA3OATP%XAC16ztw$O|cVZ8nHQZ-Is!%EXOG9&Lwj5F(GSA7_;)|Z-U6RgQJf7 zcD2uTHx=$_n)Srz`ALAP@A2qSbXemTqwCW<6I&g)4wtUO{1?EiJXpA~n0>!KCQQ|e zrAnv*lj$EP+#b-tz#%?oqQ`*@npTi?3FT{k+e;<5LAG+vI4EAr4?dJzSW&?~NV2Kp zk=I}#SG zlqX;XhdDsul2Q>D9kCkD@J*_^-e=+NzXFx|J?K^Z9i@{Ae>%$1{*MG|goZ8!9mXu0W+LXq; z7!S$b(X)kzojsnK8Glc8f|Ll7!HBos4Yg76S84};=tbqCc1?9IJDx`uy?{@t=kwla zd$~aNUxk33=Db6{J6+i$go2vEtlIRDBl}z*P^AJfu%Z8|A~$kmFpIR(U10bWQ_(ur zhEmKw$-Hj}!CHkI5uiu{{sOWkhz$%gDNpTBCyAp_cKsri{B#RT5de^-M>M%L{_!Mo zTM8Jlc26et&dSZ(;1TXjA&qXcZ!Za1aog7R(3OR!gNb6YMv$r#VW$5?^41MDfNjG%elF}J7>9FJ z{Ca8t%r<6WAsud?S*g0aLfqaOKVPAG@GgZFsE9rZ9JI2ji>8%>q5Tz*I$R+HTfCDy z9g4^<6vrc&$8UA>gz|z!`pq{+&sf@g8@m}kaPKyZCM3yf7@I|wW#o*voGd{{$41#`a1>h{xYZ~CB%cLigV0W=~h0=pDO zEHIF)?-c0cgsl>#SSk@J%c7Nsn2V1QuribxQ-n*B6p!_50*uI8j=^pxWm!MS5Q9FY2tOp$3yNMGMVzZo&pX~Kq+D`aJM*xU zg7jDK_NTn$>Wjm%+OsqrQaqhIBHM-?NkgV~{4}E&^kd zb5>hZj=-9{B-FyYAS^nfnpa{KD(zQ=~m3ateL(Vg3!c|OZhc2A1(8E#5?-SYbCCSng8l>#9 zk?)ijA^zf)!mNuyM&irB>w^KMTxrIoiAe2_n)xbxv&XiqIeOiJgmt@uztbpQl}u)H zpl?_iOotk2At&Qrut!uf^wN0OCHak53FY98rhgr@;8LqSZ5Dcwt0sYJCx5!>HWpaB*~41!W3e}5#6pOS={0xPdbIad z8P(m#_?}^ zUk#Qo2jhOZsyUW(eAR0X1OfVE{eeTu>niy`++jC%_(W7$3H?=i6|*H8>rHB9^>6Cc zKyYqkP(?O8SJQWNe^<)lL)H9EeSGfZSMll~*G9$)iT&w>^pd-sJ897|bzI8oBWPeR zAJ{8*_%d{Tx3ujb$7y!<86zt0tXXoc-kPGHK0BfuMIu=!&A&N!t2ZszENSSu#65yh z!2pTprsG&C<05}79L;aeh>WZe$_T3iSIR=?UiJP<5mRL_Ah9Fg^X8u7OmM;Hjr}ka z#|A#>qDE3#bX@Gd7oh>?ipV8#9Btrl94vZA)e@)&N6f+7J5Y?63+Rh)>>Q4G&Nu}} zfX?)QE%N?kMwuH~cal70HgJfH?Mg@N6L3x3Ih(7-x-4DTb8O4(!V`bpeiel@P{K&$ zMo8h932he|dbJY-sqfRVUrl7B9V;4cfOgzA))#G>c0U;;n2eq|Wdc0D5F~XLReF(|$O3mQtsS~cck1e41$kiT z{){?}|GDjm2z@oD*tu$ z5+DnG!;ODojW+&id6+e}DE}3CjFf>p>l>ksfxNd39~en~H3$hR;?#*(4 zd-`xSsq&=Dw`Kcj)=SLoT8PKDV&G5h%R=MVIvuKa6Z^RfnciU*ETnEBHBkorWd9&? zqIXz#XxfD-zO{d)He}Fyn%sxH7EO(O;zy*?Qc$D$sQexe5X$Y*i(u%>j&gg^QIdAT zH}U3{P6ub)V8|z^WeL?*>z(i0SHfCf=p`|W{ldhaJ>|A>Wyi4e-NFT)umOGaT_Nx9 z+`~lB5BmYTaJ+jAdm5YPKTr3V)vlGaB@+o(MtCt>&*Q5WJI|QD2BkLZ0wohMw#?sl z)&z?XI&sFm8J)Lu<}`j~p;zot@gJ)oyXvzunDQkoO=!Ry$+y~;1q&@_b>`(dB8Om` zlEwVI5%^(j*HqP#zz><*WdvzNNVUlpp-FoY>Morkkq@c$2(6dqpDmfAs*}{*P!kDJ z8~oyoB%-(m<{l`_*!D?jALxMxT7X+H4fb_-_VyiPaA^Y_>xa6*Jdc|Jrk^MGXmv>n z13rWJ@s4>j`=mB477xKcale9(jIz46p3zzv;f^^nW-I{&xl_G2ZLq96Mae#b3vs9d zkA4YEji%2g+lp~NHecZQxUu-vA6t^5l3fH1Q%EF%s~3&zi4h-Zfsc**>W&_6ug?hX z7PI|Z#J4f_mm!ktO?_~4#5KMgqZGdV9X`}M?CzcY4Ojfa&M$#B0^qr@#Lf}ToUF?1 z=cBB4MLZlZHA)vrQ{6Vgniu6J?ybW2Gs`32@beeqxXP{@sR!&Ju)eydU^p}|-BK%S zGkH4bl)v;b*JT*&b1NX4JNq!1o~z)bzx&}3M&mO*&d8NExOp#;r7Lvc4=ks6#&5u1 zMgRjl(&9X!6N1}hhJ9FeX2o1knUIL8==Hmrslm~3>`?5ECLtn2h78UeHgp$xqQv*8_gh?Bv~u?_o> zw&gQL)u4x5o0W$1^tdx|Nd*&6zNbGqXTFF%?}GYqfi+|>;N&-Toi?#V5x^t3nD#!a zFOTCmi0e-X*5Sz(aVkWP?RW3ywUfel52iA}3P&x;C>|Mrp$NBO2(N@G*7E3DMqMtZ zw7={*8Yfr~(3;_3Z=3R`E^ZqnndWHQZ9Xkg5pH{%*Ssf@`qHsiS4&<<$BmD^vo&ErSZ-m)eHd_t*t^48p5lS!YjRx$-s!(FXzM%oZIx9_TnYig+Q#0)Q zCBxlV*vXZQkrIcI#p{bU0yD+h#_zwGELS#q9h2zRMf9oKkg_sq^R2fqv|)-4$fcK0 zb0RSpWsW-9P(?89Vp}5&Qm5SU7Lz#HDVmJ@6pYBR>-xz?6F8#ZMjLZ2rlEzysKd!1m_at;i8DbVO3{rpEMszRh-`V=3>gJlmr!pUkh&RB0Ys8Wt&cUHJp z!mMg@WjPYbnCStDF2R0C>eqrVjusL~%*CnhTF=F)Bk5%P+nr>xH?^n}Q5}um(~0uDWh8!9DNdUQAjC^XXWNvJ zKSmK}9vSpL%0l;xJ&4{~ia7U7NLmAhm|g?Llo)pMbnXKUR%Zw#odq|PN5-5mmre0( zy}yiXYxpRg zF+7Jfz!0WoI>+7U&uDx@XUTTmk^$ddz=NkjwO-YEEPA9i5z7L?m@Z&%&QWr7dw5P zJ|NT^c&RmIJ-Fj7x;y#6X(y5))B?qW>4SH~>Jm0Nuj!`ko|GA5oT_6f*eo$s7JD%< zwSLFVQN0ZFN<*R2Lg2LR)4Tbx6qR6kRB{DAxAJW6ukz zwZY{67shL5KTnn|PEg_MbLPHf096)91+G{jb~_(TUYCmaKpCjcu^@uqK5f4ZI37bl zi+MlmxJlq#goR;|iVmu`A6|&`F^0v=Ee;XBbAlmIK zhDO9(Zcp(@f?vdhe9c!H?21~hv_ruHZFoh2)3*zK$T@o-dGzg>MT?{3MEkF=N=!z+ zNTlrLb`|L1_vCQwoLY8&#fRJRM%GXek9Xs(wh=jl(%nsFvc-VxnP-X_n=kkcEzl{L z_ZT9i4dKzPl~{z7CAqNmE*i!LnW+EEt{T$>f`Qde;Qu8(S}wh&UuvUXj*>0Ke|PVN!Oty+Mf z#a@-LI3|+&yNS5eO+t7rXCG$!;HK+5qT+Ixsul;!S9+*P{c)+&r3EO>Z#R)A-+jA$ zESVc|A|dm*vJi^#3w_cYCYn`4Jh{OvfN01yXOoyZAwuN=D-ZS9Js8cNtH95}_@k_n z>GUJN2r;g#A~Hdd&Fe(AHNIR2#*n~CmLJ4Wf*?hi9P|P?!V#D|p(|B+?`;{+z&-LC=1aPo0JG$Ub+zkWO1m5h@Bsl z*ssE607nz8KaBs>r`{T!lq;*SG?~mmxoDM6dVghUBEz@rm%mBr^d2p_M`>kD#Iarp zaUlg|z&YiX#IXK2d|}k;)Ldhv-S+;h1P!Zxi|kFWygb8~+jTIJg`-~@+m3Ccx`r8| z{ZNx?6bo|IZ6s#Ii%!z{7m|qU3CvXT^w1khCtSxksY}6#I<7DAf%YLdt|6O;y3oW| zpFu0;idh;EEGI6^@7z2#hlXAIfeGFy;yKi>TR&848o<bJ!=)U)Ur05hG}c0j-sa5F2agv@4FC-QR;uQW_&~d2RvJ=?-Hd#<{>iuxGTEo-bsvoT@U7PQ$v|?iywd1>im1_noinc70AV7zio_hn z`*(ftnhtRbT;2V2z>+)$OsH4%{ir<`lEhtdhr)r??D5WvaDU=)ikCSifeBa(*^<(g z*ufXZl0&5^vrtK?oD&)ew7q!z)$?${NZ00@4a z3z$P$5Iz(nnOm>~i8D{Hvx(K$azxKWHvda8NWG9631`CFcK@wPQ8HU#;=PSK6@2Bs zy}@VdJYTSmyvt^62|iFRd;K^=9ICFdG2prYM*4&fSr!cW&4C6U3#9A`Z|hhRXP(o@ zST3#+Pj!9$(U#<0ZeRh_*DwGzf!IHX!XX#L5nZ=qj1pX&2Ja#)X?bc}VBY>gukC{ zBE_GgVz`rfeA1IGFi=$|jXPik^7LI2e?nWMdjf%CWI z0q_rm2&pbYKt;B+jjIIw08|jdmNq8Zna%V;zbG`Q-bACojSIQq%gWZqQ>G_}b>0O_ z8TWz?XaYQ6jC_Vo)-u?YSc^XvotA+F6XFk}W zWvojeuWPnCe&No?4F`k*%p-K#Yp&4fv>lqU=g{q31VePd*sm5BXpb&)-LBa8&oJl8 zU++UIF1Ni6N5`~JV`wkGc@%HkYs2LZ!EJjq|6mB<1!3=C;ndm+1Q+gJm_NYt?lur3 zJ_WT6m{Ik5l}{y+-BOV;B`n6AHd7a}vLOIEbqR+n>cRc2ZIH|3ALnMOLTKN(*beWc zP<-&p<9Mr%;Oyre`Yh{rshiUk#}TM z`LeXV5~x2mNh%={~Bi47VA*)P&ng@GyGHyQm^yEC{#~9mbh;jO{2~LY-KB@9;&}MH`~D# z${;Wr67Z1h#*|?5yQ{?36$JR^({SxydnbDeUr?->QCM>gf)BsF9Bhc`=*J(@9^??; zbiTW%#quLH_5;sy8T;{#3%>1M7P^yT|43;eL}z6G7!>!Eh}x6R9`GEQo0{D&(rnfh zXyY#>YngD(sP-8>DU9QmDHw$g{=_&(XSF3C;f;O7-?=2_77e?}ec~7>1cKDnw02QW1 z>B&xJ#UjDKx1uj+JC7jA;WK+@gn28EL;V7VV!9go-b}M$Bu?63`ni zV&XPSI!W?~Iyag*NfpT)s1=+=GJ(~`!F0IJIu~r-XwnK)_UCAxxn!}Ho#F^xS{vQV z9uoj@S@^qwRYm~}AT`VU&8*POWR03gPZPdy9L(RG_0)-*9a=k+g z$H?8bAXWr-am{`5R09xh^SXo>9Mt;*q%`!J>X3c_P%+&yYe9ov@Qo`(lJft~U#>$Q-Yz1a$! z?sDkYJX>OCL(RzE!4kCQ^LJmkH>f49cpb(BNP5!^RAEkmy<%6z?JoKD!mBL>yM(M< zj<(uUO|Xv@S=hhbQ-D>##o=wAdj&E}02DX(A)3MQM}F6EMm7~PCf#Y@ za<{!hq>TCsjLA;8eHXq$KYt8x!r5lea`r5LSh>c}&~LL(^eb3C2p0GPI9AaE&AGOY zwQ?J*HQ&A>x;7#cn+Lc>_L6GoI`8VtQ8tZ!OSKLB=9)4#p77aSZ;W4j=Q-F0t|C0Y zdj!Y6SWLK{__rElJV=Ei~}O4CYpi zvtG#p7iD*!q5Bw((dRd+rRY{CE8ymqyt%cW`Eo|Taw4Tx%lqC8{XP+JRRegr5noVk z31gjp7kYyMxh>|;?9CE2_XM>bF8qSv_B5%MBw_5gfTeqcY$nG-ngi?1sXF=9yySox zo7upF!D-hd#6VKrq%TgQqsRqefmYLb%f4X*)#kK5{yVKaSbVw{gv0v%zNcjp7=cZ7 zFY_o@#-}P~K@NXl1xcPx64_Xd*?;T~2h(=Xn~Aycgy?`VuDhkxc&x{4xH6KFD?b#B z8TbU;z4fPn(U4=TL(zBVea@Cy== zmI!;<`vc{aKzTF1;sRzmG2GCeHBSDVyc(p1PvI0aymQ@*bB?rdrES&uRfO_s z=B@GwVg)zpewPsf&5>-2r$OBDyFEloE39tgU&JXCE@t%#^G(?pI1&yPJ&2<=h>SJ_ z4$MB+&bnM8TgW-`1))w~PCX-U^y#-3k0%er`^@T^e1df0v@goc+l8DFfaCqjmhu$p z(bf+cw?D>Z!Cx&N)`yRtjc+-63cE?wSMpv5K_*t2X3EaLcQJXIFpcf_u^5Rvh_zel z4WjUTm=_IX=o1|+i!8{`m8?0%euFMAk4n^%o5z~N6@K950v{2m_sgFMj907O%&QZ=_DN4D`Lbp#Mb=hxDlek z3@SIAp9H=(afJ&l5ep`ihSxcj#I#EO^7z(;e^1-xRRokPRMLD+vkf%!*M&;hIDGY< zZ08IMV?Xy4^9+y$CB6%>&vZVILK0I94FLP$(7#8e!2AB`ZTL#0^~mpu19pMNujpj( zBk)|t!%)XRrbu4#wE5Px>}A%^fb&sWenp~C=zaIlG5piBiN4i5Q)xu;d-+7}X804n z#YZgBLO$sop?PU8t(MNj4W8Z~?RvQE`zMifI*2lO17VF3b>Zx$6po6WM66}tVpND$ z=ENkA(EI{artN6+_j>VSpNDv2E+eYfq20}u$M|y}B`RIMSl{NS5)f8Bd3ct(ey{p6 zBfh1kncnv5q1(hNM+h)be{m>p%Y5e7{&e>Fxj!ofg@4t36tW@UoBM%-aRPU8O;zFj zXuvlT7~j;z;j^lHBrSIrq}Q$n#lnnNOje`0!oR#V^zGqfZXJY^Nskpr(T1x4TLFw={jR~Jr^|E-g@=fI%=J%)owrfpjSCV|N)&9bR4zxZ~G zQIW@a!w~|-hZ>JtTg=O@MBeQr%NT!5)sjXDJAVEBq()&)5v1!uK;h`5U|z;4c9<{s zHuV>Q$IXbq{P&Cu;$vlm9mx7Hv?*Ix;WI(_}^>o7+b_>fR1kA-PMFkdG zatTYA1>7mOeQ7IYB@-c@=A1HiGVtz7LJz9asr3nYBfEHn4SkFz^$XvF#OGc4PMlsj zXm$rbq_MCvWRu{@r(f1Y7#6)t(KF>9xPV%@mzgK7_8W@(kMk_ZEhFRg0VdopCu8_Q zal0##F{7`AV&vl0(LsnLp?b_N^n#})oKu1+sTau!-`3tjX_}|s%(Xs0KB#|V)v>Qb z0*AQGIjR386E(!NjrFxrhD4Ysws~lX@?3w=3U5eEbM*Cl-T@++OKZR2kOH&d|T@VpNCZgW-o4MxEDmjiSKPb*_KWtaR7Ojk}B zx8G;+T?yv3HWZN;t;*;(wSA)N{C9-KuV-gZt;oxTPG2}#R{M))j0auG@^cpO(%wmp8+0P{0#ADTsZ4+ivL_BzGEq0u#@7F+^CiX5s z&4Qc@{5uR|yYTtu9})z-tfvadgkm6Ep2cveEg#2y(-Rg?ELT_ZSzE(@W43h< zo>D0<=$a?6_})Lfy59cjB2eJN!taFx+ek0={cQw+{H0;3G7o12rJ0Y;D82*VF1W;K zf2W(|jvz#weZhmrcr(2aDpvPGlN)&bm7XLtNc)^dU@`H?Sa#b(UiQH;p}$JHHQqo% zB(5sQt2I+S*){J6zp`Iquxzev+wXvRpIx@e-d;jep&v#B0Fqx)gWbtM9hnE-K4<#A z-bS535&uVgSEPBvA?DmZWW;mkVDoe=s7|?NCxMX+64FoGloZYP|6L|3yOo7}0)I8z0}i^;u^4c)9C;d#5cjOwQp^*8N3IS+ANRzJR1;H^FgMd;pYlGDp^3q=s! z+JD?_JrQ^k@enw3qsk+9<;R9@YJS>3JctJjGdUIb_nFeQ+K+Jmjtub&&}`~ zT1b2rG@Mm4D;=}e>tONUQ!-N--%&&w<0h~gT=5yg->#s<+UUaX&OYEWCK}f8L z3+ew{1bbdal_(XX`CRb-nO>Z1QOe9teF{FXAs*>ryRwAiv=I!{$wU7@W375&{AqqQ z+(>XWR{YfN__@&ZHxZX1KiOv_<8OO=Zqv;gr?1IBGp78Nwhb>dbNgltZPRwjiCHho zJk-!4HB$=Wn(~WONL59~oBNMrl|}oxgJQD?SrOr7O%E!Qq=)mS&QmuwRV27r`4ESOdZF$rY^%XMz0j=Af zbb476k8`A($#4DFegMFIZ70ej)`{sW z-K-C@JUEH{<1J%}poOJX^bIV?UHw>+yuj%e-}~2C3IZxYuTjQ{fBnAAlHVZ{H$I!k zkQ{M${aH$BO!xlnXrL)()DbP}Cv@SLGiaFH`CE|%{ML;!RSuwVV()VN%XkcVSKDGD zkG%0qQ;MSK+i(Hec$do-wxYszy@lbD3OB6Vhp1)pj%2*F6p~*t)W2Q@wj%!e3^45z z%%@9zY}~Oi*?fi0ZAIgz?6Ed*!mpe_P3JpsJt}SQxlL)ZJ%8ot7Ugb`dq!>O#tYB;!JY2+A>2jnMY19vPWCmuk6s|4mEgvhiitJbJuwYs7r(yv{aNwBJ|mPlLDfOy(-jK)QsIK?!|Yu@D)eI zZX}q%qTS`E=?{H+)3=p%sT{pT{t_|b$PxK;jw**8tpBC0l3Ti%l zX9HOL()c*m4O#yQkRj?EY4=Z)*&-B7HK|=#BwNWB>tUuBh<~Pfu1A%4GLuW5VA_`$ zN#SX1mnUhuf(&X}v6Rs3GT#I}2^ z1Cd-c)Tx%kTq!wTkT$9AZvP>Kqb4+*eB(vW9;{elc3ev#k}oR#8$&Ap%EkQVfx#LP5QU z3M>Jkw+2?6x|f-jPoMEoTasBtxt~BbH#Hp91fYqr^Fz$_Qvg_yjiO`FNH;h1ETvhF zZ`N)|a79RIb?&|>kqaC+h;&H0MQ;+LKEn5xeW=d-0q!RAM0MFInYNh``wj@vC%_&gUGCx*Y^+Rkcd0LWI%vKKhJmh1G4yL(9{GM& zUbka_y!C-jQj^X1>aiowFB6K6*L5aUA7>ezn8}Sl_l5XST+kyiZz$JXP2AL8!0s1S z`BN;fKAc{fla$mh4tr+{yRI0?m?_H5T=}kPl4}ApMa&;U%XT0qJ!oRT6 zF&n+(bRpszCxoY)&ph`H35l}7vqhtiZWC9-==Pr}=My_n5_htfrbzg}8Pi1*G(nNTom30J-^O2XnxLIgAtN|+a_-bOc z1g7+HTzU4n({1(5?t5oeH`y6i1q*_gMTrpx_gQK_h4k+#+R)d*s7Q(H#lseV4bz$_ zER3ru#4oR$s}2R_5E5V#`|DY1;az39n6c#1#p9Mw#x3%%sW^lyknh-gpXc$r@FzK+`lnSmmr!wnVrsCPV(AL8GcIXS`f!7GwLD4)NlwrQ~Ax98L@;l z2~Sq9g*PixB(S)nnGoJsLcngPhL=5+>Nq7jR%GRDzWm(noeyWafrB+Vif`bGwtoSF zSr^59K(5S{V~vPtQSqIHhzdz9xe@42Pxhh9G`;DLQOZjke+fvx9|Hh#3tia)A6j0X zn|!Q=!~h5yXyT9NeSA6;u5u+E$o1F8-(xbcZ)vX{#%X><9w^6 z*LcqGWA1^|N-q9fv+~<81L2mZCkIP!)xoI;m3;Fz`UCwr&+kqTM^ zGdfsKabl+m|0_j?1!VYIh>>Ye9Bu`79UDHADJ<9>%lsN$MyKg4CFj&@TQ0rtr!i07Q;m@Wqd_k_pk+6@yCSP#NQ3x;$$Taa->3p+KsJwhUzypdhx`%T8Ls zp9fsiNWI@;n8m!U^!9@KIhP(I+DXEzZTc%O^h(>D(>pMgP0gP+YX6!FugR5{SelyI zevEKF235FnUIZqi0=^Mv6XOd0jeHV&7Hd1!>3vPO8U?>=jO>Fh^$F)UmOjSZVy*_^ zK!bTbZUhG{N#G$5nAYuTA;=m#o&2NtDnhT4!*~tdAsH8x`wF63hT(NXh^8hTk%#i; zjChW!!DjvpX_y%%s=4s(YbAje*oBy1gz_2$qX=ij;$`YT-=@y4K&~4boiM~DIm8aS zp1Yyx3V34V9Cr%Omb_3~J#AmoF&9SdM=7kxz!egDLtp+3+Vw-3N-uDufo&cN1|E;b zW^J7CN-gzY$18YXmEYQo?m8&@Mj0zf1X+{(Kc3DqpsHYN*qiR|?#@Fu2+|$WN_U5J zcO1IAOS(h4<4__9qI3u%DP7;W-uHgL_yPOOoIPs~duG;pp3eiD2vWlP9Yivylv%LU z%axRT;mi5>+pV5+yX_Hd^5q_(+ZYhFrqYY%ddX#GU^?(cZ6^K0qD5;H4xDg&N!!Pd zL8zL;nBRR(!yR|tFuIDC*;+bKJ139xRdRJUg;%Ip64o-`#G+4@!_f)HZwPF9c<%aR z5kwi=I_tom-xL4JkiUa-Hcy@}lugF`p-vjy{xnTWLTC0Tz^H!rZdv1Hf5!2f*#QsD zkq1e3E1u5d#$dIlR4IHw=NC)$v(^dY)$zA}a*q6gM?+xv`Q-Vnmu?pgKAMRHSJu~hNE1Y|A80eWOAOq!WjqF;|FtjO7)i@FS5~R-+jFI( zYWjMqagcaR8!0xktkBy1vpBSG58x8kiJE>-qU{u5!2%6nR{0g?MEQYF?_sG&KL;@F z_Eu^PG)zcs|9*4gwNj>i{KthDmokWl1F>6z` z*q%&TeT*?q%@YQQ&z>j2NjPZr4XZ~X7f))X`(PKNjw^E0S;I~LMdc!}&Sl>%OZDe# zxd*PE8;sL?QnC?pOD;BpK^4N*5kVRfr0h}0@4R84d=)yRLOIqyU7_*%qmL@4?X`=) zFo7Cu%D)S;a?~L@IpD(5)_5*?3>D;;^$`{o-OY$g|jXy z&ph@fR<2dU*vI5i^yu1|nNaI=+@8#e0OoCqD0y=J9#_&cM(&Eg#V(_6=2%o~jzG>z z($fHJ@+Ve+GfAMntB`EUyVoj$r79w@&I)AG5<_uvTVk|>t*k#i^Vbx5kdPtQai)Zv zJYo9a#zu8c8vi~;@+f~=o%m1K1=S8P=4gXpa>-D;HZznhucjD`g&@V**+Y77W97bK z?prPV%TPE6btYUNXWi~}ed^ZdS40M9-nX07ok3#k4$utbf*CNSHL@eyZgu3`8|mu% zSd;Hm4K#&B3P@&LmODQmK@%iWr25JvsB6=NE6q{uC&i;n z1(>)%Cdr&>k=M8|r1hy*D{FcxLfE$Z>eP~5JRlpU8dL9rGcUCtDJS9QS9)xK1yd}I zwQQyb<0{*H*%bJh37YU7MlwYRL;v*UYr6O@4P`#dYNyD)+aX>-p#{oV%lAQ4PK; zlg$OyhI^fuRkFm7c@lvWdt_}@;r-}M50ps!mgx5<)>xm{h9sa>VJg=bLU_SlX56MF)L|_@ z-KA%ByzmJzOFo~`h&L?5?FRhfnRvIx$hfw#Ne{gBgl#+gz?tB&JdjGO#B4dS{pzi$ z0vf~DguMAwU7D)&DG|8)QaVObH8)u#`Rtnp@QGtMVW++jd9{9 zrotvIaTF)!k_0-Msu{ZtLm_M#Aq;@MT}bCW`^&)iU2x$%OYBU1Q~Xc-F4uy6x11R> zfh%MeR|Gw+ty+{uBBHBK`fq<}loFn2&Jg7a;q-`k5NF^CyfIF817(65`u|#5lgo3M zx{gBAg!17se9;mvzn#GrzqlDR53}(ye>j$AU$C=8=vR z8iiK-Wa%6)_ELT5CMF?0G(`Bt_A1AihHX z5erpYnt2Bg_;v1-oV5Cq60}0wn!edkMw_cEg2C=f5RvPCMj7 zWS3}$u9EAH-93B_+e6t`B9!Jh7CF;en9Mk5v5dg+1MBPjhw&?u12^UpYyBy5XSXMV zD1&^B9f(rq6>_5+!?%SECfZ`rtjj8TlP}^5vZKheAr8LDyA|DV93{E5(B)kL-o~#_ zX2L}tYNP7&J6xIm{(*B~W+ArhK~AJo^CGjozYeH7js7-$ctHq*M|=pX_XEVUuc-Z7 zz7l451U#r1iQ(?kES4-(DRP(;zhor+(p-Eir*;rq?z%Ciy2A5kpFz2=k2!PkDprkR zFHDsH@w7$}d($5aa&_*_D1|QWD-65mlhUvn6s{=U=2Ur~$lC64qHQS><5|%y5rgXiCGPxTroABws#*rJh|T zt$3jy(G)J6mB}X?LDrrk5}VTZn)&UbiF#s&sjivCiEfQAZf98{TjjqFC{@0PzSrXlj9HcyV0L>8h2ibtv6X}@3NJ_{aJv8f;O=^rljXG{R$7#U@g zU{0OjViJRA#7qW4*$wm0>$#VP76k{djj5Jr@y!t=cLNH|svxgj<0-5++~M-VNhv>u zgl@*4$d|f9sTlOLN)WIX_>ViMb0!G_qSb||uUqT0G_u8(W#RiW zh&D*bNGw6|7AYR@;#bV>{Tm5a+W=OcJ1jW9i|e2ycEf^mC+_v3>n_SC+GEOUo{c?R zN{cGUa73gOOZ+`nmMi9~=|}LDAED%4DZHPZh#)C07P*<|ts8&QYQDDueGV|WFIxBz zA?NWTH3ARC$cC^8=;Ueq|>&C4Xp z0JPjO+-vr04^wp~Sn<3#mFEjTg|&8Pje%x=C)`<~Q$8vD-S1$wk{e;)MS98*_v2^k z0f=|Zo`U(;eKKe6C3q9|fA3leatU=IVjoz@!^_1iiw#GL?UW!Z?4$P3yHSZueh4>M zfqZjYz@RXMS#?52S1hYQ+!zTM8ivA&TtgAO6$NZ$RT?zL-gbz3B8`p5_7ybh0fzSh zNnuy;w!id|7qI8zdDhQ%VJO?1dkgQre>WH@>W&y_=8tkME)3wB8?IaX;W#O(YhlK| z^|?1My?v7baAZg#6aS8DFe^~F%NjI^K0#O_4Yv4XQ3(DccCebE*BTeyoxJHwh^aI9 zPO%r%_Qlqi|K=mPXTJ=iM$HgA`WLPB5UkcBEcD-geMb~mDc^aScWnIc2b=TBXaqN@20plbs@g@~@u~SYLj-hG_EF`ozTFi_bV> zCdN6D_JExg*P}myyVLU?{AL_nqMw5I#|syxZ>KbWdZ-3Wuf!A`xW55Y{?(C_2iss> z3CC0PMrR;)%F|IHt(7nRi6AOv(R}K>;N_lnG?_N>neGH_Pbe#B38!@>Oq3qn+f9Gm z)0gn0Xh>=a>kSpnsQ|c8fXQ9IZ>lBO(8a&Q9nKQ?+VBr4U*!cDPR20%=xHJ{Kzy0~ zkcM+tGqZ;i_Pd)?ST}NSS?YBZ4QG;9f8V8ur#CasIZc^K7r{0EUrK9?CG@hZYGUy= z3}M02bhg7l3$u>52G;uTZTEIv8}s35i;{@G*q$YK^^A6Xr`id8*&WVzq;(A|5BWJB zE=JA(D>5ih}U2YGpOb_NA zy4i0?_;IEWebE`lrh4_8vMdM)XjZVUq!HOge+;l>I@^nYa6F(eiMiom?vppfv9UBB=N zmV!b>wwrwJV!B3BL}IB!;?b+gm@)^@nTS}>iQo#;(Vr#30=Z40(!$*w9Bwei25#|S z3O>RSy1;3J{%D@WBx;(CK8(;yz=-Tq!hAw~bJ|#Pm)H1Nv~Pl}MAsR4#33wolEZsY z*f4ZXY}lVr5n!(s<&dg$Wm0xZ0UmwQnZ(wa-wojg%@c?X^XSnlcsMI*wKhduN2#wr zcoInw22vR3t8tf?ck)4nu7~uyBfrJk4z#eBd5Uep$O`V2uhO{Q#jc%GN^lqP z^?mPw{sD;X#%Wh*tQ$TgA>a92W6kbKa)$qDRAzS|idAAKIv~^aYIE4-E_%Wsl}&&8 ztQ#C{aN328>?#qM)m<6Z$n64o_ z{d{e~-cOBF!$IR@h0BVXkT%Z0Nq1`k67Jg%dSqHTmE2#VKLcTZJs95Dt(qSDaRG*+ zZ5TbR&HcmXy7$EhRtr(znqW0k5BAG>>@LpqWy_|Ra)E|dEy}pc_atbD-a{m3LIb~^h zwvA@;_ll9TvM>dR;r=Re^tw4KRA?npApt)8J}a3$sA_D-LgWQ1x^%I=AHEG{&tN|^ z`>?CjP}(Q+vX1y;j(jsSIu@e4LU#Fq&!!1l;p+b&+6&*4k`RTkTIgvN)mS)bC+5OE z<>j7X0SgFlN2=j$$fJ(f&GsM<)XjI}JerfZbhY?brP@Ws72oIYh{7z)R+x@pBcaIp zNU~6pF|mN%Enn0tGPb*r^mlyiCv$_AhL{s^aRNDLqonxD2ABGqVwVwNG`NKaDxY4M$upqhwWe0)}5&K7& zB=}yF2w{lMxgE$^&jJann9L+uts6=(R<_xTmisZ>Gs9V=lK#$ff%(m!2}yZkqAX>f zjOIy^<^yA}FbvWzNpLS(`=cuEtj7Ozd*ycXGK25z5VIcHcl}7qz|BKm?$o8-dB@Fw z?a6{do}K3%1s@}lYZB-9a?;zASln zh<^HzJ@(Fk&_-TAPhmGehV|R6Fs?Y@gNHMedxh`sg_|GVx78w_Y zWuc&~rWvBj?W06`srv2_^YFfpJrm}udtEcdA0Qk&BSOJ-?vvuS3D%zk@%matlqKid ze3Q{yldycly8Blxach0MQs4DTDfcX<-L%o>dBW;wjdQNFJ|>`=iJ?z<%D)SuRjI;O@b zO5W8JkMcJ@IlHNnbgD)agdxQSyB(V4 zE_(g=9q-ncq>krW#0^UwP3Cz08ofFbEta7({`zgWBJ8@(h8&wZfiFx)Ya6YCoNBpl z2j;_$VMF=gcxL*@ZO)M!tKRREJ>GC=$~gOrL-pM;r?2t0lEN3lgGEp&9Cem*&Q_C& zxGrqI894Yk-zxF8GO3GWMHec!r#Kyc|1Q46wzJT4ZX1*Coy9pqN?xCrn^FO&9|_kw}%8r8B#6|E?} ztu!M%B>$(NagBX>lDDj<8!CgnM#cDbjmV3U=v6d&e!VFDH`)Tol5Axe@EB$LUi}dv=qsQL8hNK(8`Fv2Cs~k(KBe@t9H*1mOLcYrjm%1xSdZ^t zd_(sN3ymY=bk;4QM+}B0-F!~}>YPjC5KL*&52{BLVk$)v zhWK=RK0(a3#P?@%tHDwkpe7>bVHJMEl1CeqImJ>amIgrc1w_myj7=|Jt^Erh+0trD zt)-M9#YXeaBUK264gLcBiJeI1{Z~r6h1KNq^4F)vI96lgrn~%%f0V*KM$|Jgg2qXA zUrGDtEjMi*ahB`0U^usv>J;?h_gq^e(=A_$!Lj2~DV_~G^z)4^ES+6)ELz5C}TwN;54(kUn zyt~MW0%ERl7+g{N#0+2@rZ6yjXj#FJoyUI01mc4=#wSh4YqMHqR-VnxS}m=fJgWik zPGW_b?~_4AdwA(I%yI<|UAIft1Na z4-1RFRad^W?2!N`>A8hOIg`uR;JJ1vk%Tb5?Z!;x-z!*(BS;Eq;?~YS`;wut9_f_HC$QbhKVM%FZ=lbFLoSL0Gl8r)D$=KR< zMC04V1D!SLCEP^g7B(-&X8;!6y^kPB{NV~r@307Dsq&*%p(?`1Yv1<|+Ij9uB^ z@y^>S32yTwg}ur9^3(NpOb#ZPU3OfvIibq`v!MqQhj}&z0qh>J%&MwT!vHq4);FcW z`75qkvYI-jISGMVM+C03T2JC~Xr$(;JIrS?cc?x9S}lGz+PF=hUFk?b@& zQgH)Q@`Mc!er@HDA}zuT`wnsf@gGq3%|Pe$5VFhFa*-4HtqkP0iTVRCqXB8~hi1FJ z(wgSy;1h)s$mHIo*gKF!XpI|cq!9~$TWGLPhb-Tql0?1-D;jSXMMog4VT2;K$dy?7he8_#MkZ0u76{K z(6EaXa9k#K&cbI!?|tFWP&#x7p1e<- zH)F$i79RxWQ;RwdVMMzA&Pjq!hwMf$C{H3Mw{Uk2vh-s*nFV#*r>v6Yog4T#4@D(f z!qgPhl15&s%thxx(|Ud^3fgj)+)Jyf49obuS;`4^ZN0@<7HjehG;#Wq77}SN{A|#~ z@O-5iryZd4ks3S<-4eowKZiDl@^s@F2Ib5&%j-QAQKK)A1BB>XL0uwX8Ug(p$* z)g#cU6myLv%4xq6gaj1zXI$_-q<}L0P;uY-M&I$dakax@n>2ir ze>A=!0TEhXhf|VWB-P6X+>e7?C5e2$NaS6bLp(G}*?@(j*>PUSUGi^5!!?e4#+nrs zYXXC}U=F)nrZ-Y==J@KsK^_QOJb829AqyLU{jgh*wg<4)ZZOh3D9(5@A^_fQ*p(8> z&rdSo#%P|l*=I03zW^c1@XNCAxp9^hV5O=B{bckzr)D$%5b*MS%P!5bUNXbKj8ls6 zk~b@%7@(s=2WLc?{osVb-txnI>&qA~G`268F?>Ic0)k)~;gT5bo5x{{YpcGg4>|$g z6|fYZ7SzmGX!_k_I=lH=qPjb$3;wACe%bRWb$re|EvDGzhy=#jpnGUVZXLm686tZV zUy_5z@QKZ{p!;}MWufHe=A%8EYTbmCf;SKj!jV+Org_)4@chHjWq}Z35SlRx5dCYO zQ+fPVtg34SJ@p>Ud7K9~od#+A_K{6mc2uy~Gn)gHd0(}U#ZtnZC8-iGTt z_ExeMp3_^Ic}NI~RlFuECP&jg@UY!iOc-RSS<9xc4CFYxC(j5&H;Y9W=zNBy&LhG& z!~(5k(Ar`6rn)Puo7(@d_*yW|`^Q~;#E6daFga3G-57xHN`KbW3u}E(el};Fci}f~ z^mk3ry8pt*0xu{^mqLUt+hM4uNgH6#0EDDTUlV+rFq&uN$Ad%U(06evn?b~IvxV>qRQ z*t~)O1B93LAbh%CZJ(`$dEn-q7HZE;6StSdA`|<_b|lSv@b)R7ueq{Wly~rblxi`5 zFj?34igzK@;rFyVc@18&z+n40K~wmTqr0W*nQ%?L?GrrU8G}uSKXqdE-KPUF2hE3G z7a2eqPa@7e^S$x!@4Y_~YsN1cz@qGkW|bF*%yXafjN5*H=}xZ5v^!(Uw*8xh3;!^cqe>W%kU`Rw|&dyI2ilx~#HIPc$A51ZGoKJX7p36~+ZDrOuhSt6rgFd%b$mF#F>i=1a_n%e#Ib$&_OIsAQwk2562lBBo?`|q$KC$ zdSB*{YGLd=4{=t;zv>=LfcSi80fA9ho3W>jZQE6RIr$$qdb4PIn$WKqa-fn_lELXJ z#_NYlmqW!*edL6too3Cu;_5_oVLck84I$`vtwv9xIS9h?78XmbbSrdQ`b}i)wA$i$ zh$_X3MKW=O?Qyifn&&V{Rhn(@1*OAE#+tjH?(qPwx-U(KkACzj-0a$*$X+>1Yw``G#F+*~uk0{@HIWw)a`sW15( z@j;2VR0kjc<{g1-5l$h#0(43)Y>wTW$| z^+_yb3tG6=>9oeoP$c`m5s~#>+J7qO`RlFHA+DWdDsJO5k{;|8{IL7= z8jBo4jrA6z*I*t2v;7i06drqP}q(fs?7o6APn|y1+MkIqI|i?ke^UJwlWt=O$P% zI48|99~ExT9Rm$x$%F+9nyZ1Tzncq$_=({y`5E!YcZG@>#kU{2zgIC>V7sq9{Kyw3 zMj7p~^Gj)HV=2b~^iu{!eQ`~$o2Xc#6B-6)pM0_ip9;16^LCSDQRi(b9kZ~!=D!da#LOJ1 zglHwlxkZmr{bMfN{ExKnKIr&rjxqgh7vG=`-w-3LIK2~7+tC>jnU@!s;)LFiJE!_3 zfBw%tIop&`&*h8hymxe095&K-m7BjN`q@-J8j@RmgQ8(;eG_MlBJ0#&ECP| zaL;G(`QGqGsDc`&>JFSmR(Db+&52z|i2~Vt9%}9Lof`n4?O=r1!X4|pQnmdJ?4CQK z%NYHm4ETD?Z)s9U?$hf%L>NRrUMfbDc|m6sh6}gn%6KL_C5^vxHF90_6CYstK_tVO zYff1ekD*Q*uNhz~W`g3Ks^&idFBNhoJBvtgKs~=S!g$3kg;;P;a#hwfdgCT&P)>48 zka4A*cQ6cCtm0Zwb%k4Z7!cKX?s0((5PuUF4)zJ`?pDJxjoSCC%j6Z>Ci{VR1{`Cx zY|JR1kr#d@YnUad{uaXpOh8XYstvgXy`u{1JoZ$eXZb6ngYkBZ0Ze>C>h(Q$l-c9KEnZ|lvyTHHwNy*T2V z2nxFlwC=P-l=2bu)Ct=Uf|*R8d?%C#c%UU}z(SZJsO<@R<{kzCMJ$#b@zODUdLr5< zUBy)ckm5WW^&F_}VjKF355r?SJNA^tuc=4uebqX$GFgP7C%6(NCMD0FY$BI1B)OBF zGnLS$;=>`702?%vn!REef3GO*Tm_V^k&gH(k<19AG#phX&rCff$7;5+z2i;mhY#t9 zy06c7a3TRQ0R5$y1CrD8y0e%c%+urdIs5G%il)i}DBHXBS+KR{qt%}&X9&FL)X`>q zQfHVI2GLa^mZ#%qNhnC4%n!_5-n-vK7s>J~Boc@^_=a8C*=a^0OqSvaUcLF+`tD*8 zgwXke5Wu#?Mq`KXA~!{Zk{Y`S`z&g{mSJ^XDMG$Tj?yCDkH@@Jj8PYJIbGVUi6U;= zn+3L{5g6HA46!JCp^lR#wH<3Q|p*sm(9r8OggpeUWQU8jbU0xKem5cFK` z?4k=8Ba5bwtT}JbMxg5#l|j+igZ4tu&o7R2Fs}H%KSo&Nn62@+xTn+VzVgFJCUHj( z3(?xCj`0MY@KPCJRE=|A7~G3aY87s|Cs|6dir{I zQ~g_pVjvf+!TQk-V)tX1w+?KUT4T8**n-*31yE1A5*i-oEIaOt_s?S+ra16nX-`pS zb>xo)20#vo)AlGquhI9+gN^c#jVrp}mPZnfuSZX-T8IzWkpXj?U76apZ!1f#k)YQQ z=Fh2$O-*NSc*eZT_fI#oAp{IC!A{=e*o0Tt4c$jinIQaRGOH|I5nsEO#MV*(XBX{B)lt(_>$=Z%?btCP;!V z(a;w&Sq<*tQszUfMU|?@Mq871V;qIbK9;d8Rj!8Mahz1L3h6>dyic+^*w>}sZeGkr z+0_Z}L+l!Y!8Gp6EP*Ejmr|Y@L%d&`l~CuUx`c%{xTLJ2SMIPeF@IX9?EDFdBa!qf z+;TCh^&^KB!@HeA&A zps(SMrif|T6JD6zFO=&GIgSF&sn3d>v8~+&pLZwO-Vc2cXa;kfSQ7cK%Fw)nS6~Pi z*;|ZZY%39`uq7y~Y6`>Z&a#R~!U6DDV$PqFy20kr^n(g7Nri;7f9!9dG`l)03ZvB# z8ceqs9<52*_@1v-D6=){Fch}VZr#EVqNjYjh=3($9-l?&=7*ogvrPStuOv_Dq8xhf zm~F7g^ydh@nzF=CDwDBczD&_3Me&t`>i^}xW0w_)jl!NXu>D+bPN%8z9HA?hQQU6v)r=1#b~Wo$2u;G}l#uRgM?aIiW=8%e#mf ziw?0|o>=Gqz>r;)usO5Av&6F_5KDFPNJ(=HVEXFl=cdJPrHN*vF~TdC2I@_u46;;j zf-P$cfkgedu0PgvYzA|vy3LshC&iQIH&t+A@00}?xO4$Pix{%kb*1-CdziDMC8fR2 zlK=PGFlQW|(1uMWBe!=xuiiMh5cWY?W5>tGBo*f*J334HU|W>WtFB$wjn;7M%;gaG z;7^vrow|5GF=0M|x|t^UWg?l&k|`XmDcB5#NClcQ-lyxM0zYRP6LZG$-@^da^TF(G zyJc)ECl+L6`Xg~8E{!!Ai ze`50=VmoRkRS>De+MvLV8*@WWMLLmV9SN?bf(prsJ=uhJYtF|ybB0=j89#y6JUF-O zWP7MJ$Ys_bOWp~Tf!~}}YGHUZK`31Y{gTFXJ=-(DaB&!Zjuk?6T!!pPSX zy+2hbc>E4`1I6D9W-W0lcJBIdsn1)5meuoq-^2{k{>>Vp2MfS@?k{J<(GDlxoOlbZ zd9U>qzeA73PlEsWwD2v|{m6LOeIg&fd1|w2>UHI#FAY#k#4GMgNF(6>Td?mPW0k=hoa5fyWmm->fAE1GZm; zSyw~3fIcJ476MDO>g&--hZ9*H3!Da39{kMFQ-VRj5Sy{RcD@~^_IqNv`1{m}F0fbO zp`!YDho9lszY%KB?^pcOX@8NT9h@(}I>XSkafpDJId@mV1DlP=p&W5-FAjiR%*!>QL$Kce zuHmA6g~Ur9kO~aK(e6=z<|Yz6O=^5All@9f?T-NLU5M-Y|GE?!MasAKH)x>KT~qA+ zGh{dqvtg|;ijNG_5L;Ho0zcVV8H^Wog}r3V_v|2&C{7a6!+Vx}I*^6t=*wBPmXE^x zebA5&jM21?X^EU+?O8#4HvRgngXN&0))F}4x@OF_sFoHR1pJ)c8yiYjOOb<*Z z60P+S*83#KJCYcm-PoEnTk~(ZL3cwM`)XcjG8&!|uM^dtk#YvtKq?^!br+!kFu6@G z;mRQ0kt+KH7LfRH^XSw=0{62}OIeNkAPxYO!fY#BN z}j-TTFdBobjy1&RYh102#YDgL#urh%uxjkpbl7F@Vyd&nYx?x$dwi zpM2zn{T4a~sP)~MSR6T)i$5us?hyK>T24NpeF+x?_np`x4Vyo#e@9X~z+(53!hF}7 z;oTKQSeW2#@OK|4b8U5?c5QOTnDCqU@;)Qs#Olb+RU1N89V^Vb3gu7Y5rv+V-)Y6E zx7`BUe$IA3rd%AHy3EEs-VINvZxcOj5V`P;(yZ`=U$C22CE3?Lq(IFbX=dPZW$E@> znG;Ki-RwETX%r*k@<(|-YH!{mR`o(9k$Rz=MfBo9#47490Jazp+ zIaa4rxY=J52-wY75NnKRG0HP7yD?YJZTwvNJzOYl_ljg5eZ?SmGqJ#7 z-5o?kV*Q=_QGe93vIX1+Dq;wS!P%F_X%)|vNz;I$Aj`;Xc;Zg??k2;H=L(+aw#z&Z z{=^`ZDR(8_;!4N_ZwQu)dE{|h)mb6#>JQG9p*F>T5778{G$|Fe7)tn+j@V-pU31!6 z_Y#L!U*6zLPG{vJb)=l9h0wFDvPTPwK5^r)6v5-_kYOcH-g0_1#)Cd0Uzwnd8&AER ziXjvmtitcVI&93?lC7#^==f?4^lBrYy<^szV1yWl<>U5JK7%QA~_Gh47Isg~9eL#IEp_EN+8n&l5m(*2AlF&NnSN z{Oo3QqN|Pk)#biRP}(%d`y(|8j2d~$wP@9nK~SX$%j4<{VJ3GThGP}8+Q^^_?AC9$ zVw!qeb-kX&e7~=B(GUyhO%T!;gSlxgVsEz4Qeua+*Nm) z%)W_Zu@@A1WUBhSNp4yncafPM=3>2nDdn;O$1(O5(d*X*&KB(YcSQjB~@02l$^3F zNi5^*$a7mn!HKo#2zb}&vNj;X@Nq&|ovWPIn;Ga-f`twO=$@L=(Jada(zB1>U zrDlt39cf~yh7gIx;g@C*?S=^n7@=PYc zBI=47O3G>C`M{n*A8b0(_rf@RlZI`DjYkqweg4%czauRy-Ya4i`*mSrb;Pi*gO^6Ua$$0m8!2Ql1 zXM#?97@C3q3|_+%P$}EPUlr0NDryMq3gc6HW3k0omwk7-G!~U#Dhw)?U1u|HR*T`3 zKL$lAd1LQSlK*S-6MBJ{LQVKJg)O%biKvMGqB#~3(HXYyp^7edpTZyFDy=6P{u8In z@L5(s{AdcAkb{_(GqF-0Z~0#hHBQZlm5NzX8UI=S{kl?8M8&*|IlCmzjyP0B*MS(2 z?>R4|f?{&1d|X?GOk*tL4@6)yv7cIk6?aKr#x+P8%|5PwOUVkbkDEKqwH-;`m4LtaCu%!oE zcskiXVeF0%Ht@`>@a3?TZ5v(cPuI$@HYRtr84BqTn|^+|x@Zzeu}kj%ytd~Yxv44T z^M`0F^mhS`EnBZh)kWu|gF%c;P9)_wxc`--O@srh8RZDmBVgt8>Gd)4*WifJZ@{OV z!tys#P4ADS{^J#;NkoF-&bwB;_w)jB3{^QSzhY?=Ip7(?DXdTz<4P!eZ~rG9Lq-WH z)1Mt!%ZgAxjECn1_B}q5XAr`oTiHrG0Ff#stVe}j2wdC-Z2!OI>OGR6Ve;wJT$sW(vG|0&l5Ge4Dk)7V$k>M16ein`2P~(z#E}F{ zW}PT)9sTQh43wt`Wg<9QUtx)Wz&QDD>AnQu5vJ=S7GoM0CcjcCxKc^cED;x;^RLPh zFw~if;OP%LM?U`ZLFQM(7Dt&MfRbJp3Te=*ux2OEjS+Qt3uREmmS^x{pe>s_05p~7 za8z|LI$D!jl~*h?t87dE6m-(CTgXUvsa(EM9q5)qfp&_O(LAQ2t+>dVM}t=7>#hVc zB*Xr0WV3AgCp7?cHzq!N9A8(H`F|G|3t+Z@t$N=&Aadtx+o^Ui+`K2x)+%O%&%IL~{YSP&`^@_y}rU}5VLZL9{ku0vMb!kX@C9u1UqJ1pjC3 zkWm1(^ybX=US-|~?)0T2DN2@xd|%PlT}(h}SF2~$i=q1mL?>?N=^B;rGd(a&Zi!aJ zn_=L%9U_wXpYV@>m+DP#O-TS(q4W_$s700r-AJ0KL(!$Yp!^zZ3{%Ja{=b1LR@Ea= z5v$VbppD9|M~BuYPKJNNKp0ab0!K46JUazJL{PdjpO*@v4iCtg%8N8>_17!Xp(UL>_(wL7BQxUY)6sbA-e|@JZjmeZl-{F)nXUm9idRT0Ya z%#8Q z_bGq7(OvI6Sna%#GRh(Nb{z9BMxprK2wFz%!UY^J9&t{*&H}cV;1ED&*BDMe#g(yG z*ipm#e|^N-jtCFgVS~vW-sil3g*o;JGRT0$H6pVxYT*;)zq!|@$#NSN^(dA+k1m~Uh4h{^iG zO~Q^kHvemv$gGk98y?>zV_l%(;dNU<)%3RmfON;~OcjJcrByliPyFd7mh5NzbJp-+ ziLt>X6C|7n6a#uaC$G`D{2_MuhMDq3mmbFxAIE1Qn$YHVgvxRQaSjLJNUtKWp%!^* zlr-UZ5w|CIJ!2RtCyK^bqt7SAz`0; zX?iTC=1^Di;Im|ld*e5!rL+yL3=c#*vqAy)Z%!HTC)P0CoQ&5K|NSqFz(S$4xou7K zb@REL$r**28L*$>FZXrNf{Ef?ZPBDO=Xu-_T4$8_X5RJRxg$i9#Y~GFt@CrX_1Y+g zyKsT{=lMH#7>;;0yao1-YUrB-be@2E$uoSWLpeW6sSN;#x8paPoVy>+_gcvjuvW~3~E=88QrDTmJ5-wK=B~68BT+0w+30XtNmd2FK zps@|wEQu^beB>quSq5`YbEf(y?yu*(=Y7xfyx-^hJkR?%=kv|`+iI^R2RK!ay$F_y z#GDfc`2k*1j(YCsB_6{5SMZhV-C8&|dLxsjvzrzV4``T))62_VSgu?E( zL0i)-{y8Q9E;ldPM7%}%j9?oILd^_`F1xo;`M{PK(&wOFko|RDH_4^UZ&pI}LdTBX z-m%!Rs2JINY*Yf!@jO+-?*q%rSK8;XUDka8e~7SYH)uY_U9Uz&w!Hf;dGlfFp8l zrEP*vT@3c)0?o7v8yodgC#HB!FqxU=#S=?nCiPDaK|8+>FuEjI2g`GY0lw}1Ma;E; zW16p9h!vS_vaK_7_^MvcizU0_K@e#ww)r-Xhk;(prRoBwMUsEnhkG?7xu^rK_g6zY zYM4+Fz1om5q;*y`@gJa2?F*0cMcUiPqra0jfYeMkFv?-(yN7-3 z5#FS?(Q|9~f*lr=fZiHJ(){*-Z2VSE*-?%|GK98QV8PFFT+pDFEyJdqj@q7Kh<|wL zzbSW+o8ZOGoZQVN@4jDSy6!<+df;Xh+^*k;y$co$F%OaKrj)5#bT>J`g|=a~OxA`k z*zBScYDMw|1{XYY6g)CwbTQA2!V)Nkl)dRlEIU3#niTGQt@%`az>O-_i#2RBrf8@! zS+sTB`8V->(J?mS1aBke415NFh}xB1;3`T)15C zkwtupXO-SA9y#VX?90dR(|tby{D?;v^L};jRIr3xU33`f)`qVdJLKqVi0o;U{72mS zRYF4$#jV_p`UWa7j#pGNK}Y<8nMZ<_{c z%RIwLiF$Y;DP2;K(z?iP1bOZ3z3vY$zDTbNmKu0}_nA7{KS2TUvI72%3u3CEmUrnf zTKJ*iHHp$w?Ztk(UdP$2Ba}p|v32!)P<3zdcr&h6S?Qa7E2p7 z;>meP7Cufq%XU;ZYw&u+E%}QPA)#Q_l}8uGVeW^##zM`e0{pWMvcS`=qap!8n*_3f zl?F^unnh!8<#PYeA==>;XA7Ujv4Z%9Oh{dzZYIs(x4vtYw2r|aL-Yb3E=y~zST?ph zHEC^=m9pz1^60JgiDY}b4gEpA{ZOlxvfMXwMx{x68@s1{-DK1uyJx7eeY?uS6nT2K zAcHMqE^qNKYQ_^@jqPvjEVfQQLsp+*@Toe{Y-Nj6aTusE2~<>kW0CBQ0*{+yj^k8& ziww=cvzlc0JP0XJPJrA9kqJZq|KH_!W8ol6I2Lu?=eU?s9@ql`)|U1b1aptu{{ihg BaNPg^ From ae8c01eb9d07b447390eaffefe290ab79ddaf8ea Mon Sep 17 00:00:00 2001 From: Csaba Kecskemeti Date: Fri, 23 Jan 2026 19:01:29 -0800 Subject: [PATCH 3/6] add link to playbook --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0452d66..2e10a9a 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Each playbook includes prerequisites, step-by-step instructions, troubleshooting - [CUDA-X Data Science](nvidia/cuda-x-data-science/) - [DGX Dashboard](nvidia/dgx-dashboard/) - [FLUX.1 Dreambooth LoRA Fine-tuning](nvidia/flux-finetuning/) +- [Heterogeneous Distributed Inference over RDMA](nvidia/heterogeneous-distributed-inference-rdma/) - [Install and Use Isaac Sim and Isaac Lab](nvidia/isaac/) - [Optimized JAX](nvidia/jax/) - [Live VLM WebUI](nvidia/live-vlm-webui/) From c2829a54210f1af24ce9c12d106e343b401e9614 Mon Sep 17 00:00:00 2001 From: Csaba Kecskemeti Date: Fri, 23 Jan 2026 19:05:48 -0800 Subject: [PATCH 4/6] playbook rev3 --- nvidia/heterogeneous-distributed-inference-rdma/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nvidia/heterogeneous-distributed-inference-rdma/README.md b/nvidia/heterogeneous-distributed-inference-rdma/README.md index ea86d4c..44bfb8c 100644 --- a/nvidia/heterogeneous-distributed-inference-rdma/README.md +++ b/nvidia/heterogeneous-distributed-inference-rdma/README.md @@ -61,6 +61,9 @@ GPU memory → PCIe → NIC (mlx5) → wire → NIC → PCIe → GPU memory - One QSFP cable (QSFP56 ↔ QSFP28 compatible, 100 Gbps negotiated) - Direct connection or dedicated switch +> [!NOTE] +> **About the hardware used in this tutorial:** We used a ConnectX-5 (MCX516A-CDAT, 100GbE dual-port) on the workstation because that's what we had available. This limits the link speed to 100 Gbps. If you use a ConnectX-7 NIC on the workstation side (matching the DGX Spark), you can achieve up to 200 Gbps. The setup process is the same or very similar - just with higher bandwidth. + > [!NOTE] > Interface names (e.g., `enp1s0f0np0`, `rocep1s0f0`) are system-specific and will differ on your hardware. Use these commands to identify your interfaces: > ```bash From 557fba70c8f514fa67d4008ea42e09179c553d34 Mon Sep 17 00:00:00 2001 From: Csaba Kecskemeti Date: Fri, 23 Jan 2026 19:09:38 -0800 Subject: [PATCH 5/6] playbook rev4 --- nvidia/heterogeneous-distributed-inference-rdma/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nvidia/heterogeneous-distributed-inference-rdma/README.md b/nvidia/heterogeneous-distributed-inference-rdma/README.md index 44bfb8c..4b0621d 100644 --- a/nvidia/heterogeneous-distributed-inference-rdma/README.md +++ b/nvidia/heterogeneous-distributed-inference-rdma/README.md @@ -349,9 +349,7 @@ This shows how GPUs and NICs are interconnected via PCIe. ## Step 9. Connect the QSFP Cable -**Hot-plug vs Cold-plug:** -- Hot-plugging QSFP cables is safe with ConnectX-5/7 hardware -- Cold-plug recommended for first-time setup +**Cable type:** QSFP56 or QSFP28 cable (they are cross-compatible at 100 Gbps) **Connection procedure:** 1. Identify ports: DGX Spark has 2 physical QSFP ports with 4 logical interfaces From 59bedc4afe767c88987751134a99255b7ebdefac Mon Sep 17 00:00:00 2001 From: Csaba Kecskemeti Date: Fri, 23 Jan 2026 19:26:14 -0800 Subject: [PATCH 6/6] playbook rev5 --- .../DISTRIBUTED-INFERENCE.md | 34 ++++++++----------- .../README.md | 26 ++------------ 2 files changed, 17 insertions(+), 43 deletions(-) diff --git a/nvidia/heterogeneous-distributed-inference-rdma/DISTRIBUTED-INFERENCE.md b/nvidia/heterogeneous-distributed-inference-rdma/DISTRIBUTED-INFERENCE.md index aaab6f7..7d87a18 100644 --- a/nvidia/heterogeneous-distributed-inference-rdma/DISTRIBUTED-INFERENCE.md +++ b/nvidia/heterogeneous-distributed-inference-rdma/DISTRIBUTED-INFERENCE.md @@ -62,6 +62,13 @@ This guide walks you through deploying distributed inference across your heterog - NVIDIA Container Toolkit installed - Hugging Face account for model access (some models require authentication) +> [!NOTE] +> **Why we use the `nvcr.io/nvidia/vllm` container:** This tutorial uses the official NVIDIA vLLM container image (`nvcr.io/nvidia/vllm:25.09-py3`) on both nodes. This is important because: +> - **Version consistency:** Ray cluster is very sensitive to Python and Ray version mismatches between nodes. The container guarantees identical versions on both DGX Spark (ARM64) and Workstation (AMD64). +> - **Pre-installed dependencies:** NCCL, RDMA libraries, and all required packages are already configured. +> - **Multi-architecture support:** The same image tag works on both ARM64 (DGX Spark) and AMD64 (Workstation) architectures. +> - **vLLM ready:** No additional installation needed - just pull and run. + ## Time & risk - **Duration:** 1-2 hours including testing @@ -310,15 +317,18 @@ On Workstation container (worker node): ray start \ --address=192.168.200.1:6379 \ --node-ip-address=192.168.200.2 \ - --num-gpus=2 + --num-gpus=1 ``` +> [!NOTE] +> Adjust `--num-gpus` based on your workstation configuration. In our case, we had 2 GPUs (RTX 6000 Pro + RTX 5090) but only used 1 for this tutorial. + Verify cluster formation: ```bash ray status ``` -Expected output (should show 3 total GPUs): +Expected output (should show 2+ total GPUs depending on your setup): ``` ======== Autoscaler status: 2026-01-10 19:46:26.274139 ======== Node status @@ -330,7 +340,7 @@ Resources --------------------------------------------------------------- Total Usage: 0.0/68.0 CPU - 0.0/3.0 GPU + 0.0/2.0 GPU ``` --- @@ -480,23 +490,9 @@ vllm bench serve --host 192.168.200.1 --port 8000 --random-input-len 256 --rando | **DGX Spark (Single)** | 213.12s | 105.10 tok/s | 132.00 tok/s | | **Distributed RDMA** | 191.09s | 205.83 tok/s | 259.41 tok/s | -### Key Insights +### What This Demonstrates -**RTX 6000 Pro: Clear Single-Node Winner** -- 5.8x faster than DGX Spark for latency-critical workloads -- 6.5x higher output token throughput -- Best for: Interactive inference, real-time applications - -**Distributed RDMA: Aggregated Capacity** -- 259.41 tok/s total throughput - faster than DGX alone -- Combined 224GB GPU memory (128GB DGX + 96GB RTX) -- Enables models too large for any single GPU -- TTFT: 139.94ms mean vs single DGX 213,120ms - -**DGX Spark: Memory Advantage** -- 128GB unified memory enables larger models -- Slower inference but handles 100B+ models -- Best for: Extremely large models, memory-constrained scenarios +The key achievement of this tutorial is successfully running distributed inference across heterogeneous hardware (DGX Spark ARM64 + Linux Workstation AMD64) over RDMA. The distributed setup aggregates GPU memory from both systems, enabling models that wouldn't fit on either device alone. ### FP8 30B Model Results diff --git a/nvidia/heterogeneous-distributed-inference-rdma/README.md b/nvidia/heterogeneous-distributed-inference-rdma/README.md index 4b0621d..083176b 100644 --- a/nvidia/heterogeneous-distributed-inference-rdma/README.md +++ b/nvidia/heterogeneous-distributed-inference-rdma/README.md @@ -128,7 +128,6 @@ Both planes use the same 100 Gbps ConnectX network in this configuration. | `infiniband-diags` | Diagnostics (`ibstat`) | Package: infiniband-diags | | `mstflint` | Firmware inspection | Package: mstflint | | `NCCL` | Multi-GPU collectives | Built into PyTorch/frameworks | -| `GPUDirect RDMA` | GPU↔NIC zero-copy | Requires nvidia-peermem | --- @@ -546,14 +545,9 @@ Example successful output: **Performance Analysis:** - 11,664 MB/sec = ~93.3 Gbps -- Achieves >93% of 100 Gbps line rate - Excellent! +- Achieves >93% of 100 Gbps line rate - Link type: Ethernet confirms RoCE v2 is working -**Performance expectations:** -- **>90 Gbps:** Excellent - Ready for distributed AI workloads -- **80-90 Gbps:** Good - Sufficient for most multi-node training -- **<80 Gbps:** Check MTU (should be 9000), cable quality, or PCIe slot - --- ## Step 13. Configure Environment Variables for NCCL @@ -581,23 +575,7 @@ echo $NCCL_SOCKET_IFNAME --- -## Step 14. (Optional) Configure GPUDirect RDMA - -**When needed:** -- High-frequency GPU-to-GPU transfers -- Zero-copy GPU memory access -- Maximum performance training workloads - -**Configuration:** -```bash -## Install nvidia-peermem module -sudo apt install nvidia-peer-memory-dkms -sudo modprobe nvidia-peermem -``` - ---- - -## Step 15. Final Validation +## Step 14. Final Validation At this point, you should have achieved: