From 24114f735c0006f17e5c201ee251b867fcd3bd89 Mon Sep 17 00:00:00 2001 From: GitLab CI Date: Mon, 6 Oct 2025 12:57:08 +0000 Subject: [PATCH] chore: Regenerate all playbooks --- nvidia/flux-finetuning/assets/README.md | 11 +- nvidia/llama-factory/README.md | 4 +- nvidia/multi-agent-chatbot/assets/README.md | 1 + .../assets/backend/agent.py | 2 +- .../assets/backend/prompts.py | 2 +- .../assets/docker-compose-models.yml | 4 +- .../frontend/src/components/QuerySection.tsx | 209 +++++------ .../frontend/src/components/Sidebar.tsx | 6 +- .../src/styles/QuerySection.module.css | 124 +------ nvidia/vlm-finetuning/assets/Dockerfile | 19 +- nvidia/vlm-finetuning/assets/README.md | 17 +- nvidia/vlm-finetuning/assets/launch.sh | 9 +- .../assets/ui_image/Image_VLM.py | 339 +++++++++++------- .../vlm-finetuning/assets/ui_image/README.md | 197 ++++++---- .../assets/ui_image/assets/inference_page.png | Bin 0 -> 771128 bytes .../assets/ui_image/assets/training_page.png | Bin 0 -> 366419 bytes .../assets/ui_image/src/image_vlm_config.yaml | 11 +- .../assets/ui_image/src/train.yaml | 9 +- .../assets/ui_image/src/train_image_vlm.py | 43 ++- .../assets/ui_image/training_logs.txt | 84 +++++ 20 files changed, 603 insertions(+), 488 deletions(-) create mode 100644 nvidia/vlm-finetuning/assets/ui_image/assets/inference_page.png create mode 100644 nvidia/vlm-finetuning/assets/ui_image/assets/training_page.png create mode 100644 nvidia/vlm-finetuning/assets/ui_image/training_logs.txt diff --git a/nvidia/flux-finetuning/assets/README.md b/nvidia/flux-finetuning/assets/README.md index 5aada93..c7d1b9f 100644 --- a/nvidia/flux-finetuning/assets/README.md +++ b/nvidia/flux-finetuning/assets/README.md @@ -94,6 +94,15 @@ For the provided prompt and random seed, the base Flux model generated the follo
Base FLUX.1 model workflow without custom concept knowledge
+After playing around with the base model, you have 2 possible next steps. +* If you already have fine-tuned LoRAs placed inside `models/loras/`, please skip to [Load the finetuned workflow](#52-load-the-finetuned-workflow) section. +* If you wish to train a LoRA for your custom concepts, first make sure that the ComfyUI inference container is brought down before proceeding to train. You can bring it by interrupting the terminal with `Ctrl+C` keystroke. + +> **Note**: To clear out any extra occupied memory from your system, execute the following command after interrupting the ComfyUI server. +```bash +sudo sh -c 'sync; echo 3 > /proc/sys/vm/drop_caches' +``` + ## 3. Dataset Preparation Let's prepare our dataset to perform Dreambooth LoRA finetuning on the FLUX.1-dev 12B model. However, if you wish to continue with the provided dataset of Toy Jensen and DGX Spark, feel free to skip to the [Training](#training) section. This dataset is a collection of public assets accessible via Google Images. @@ -137,8 +146,6 @@ Now, let's modify the `flux_data/data.toml` file to reflect the concepts chosen. ### 4.1 Build the docker image -Make sure that the ComfyUI inference container is brought down before proceeding to train. You can bring it by interrupting the terminal with `Ctrl+C` keystroke. - ```bash # Build the inference docker image docker build -f Dockerfile.train -t flux-train . diff --git a/nvidia/llama-factory/README.md b/nvidia/llama-factory/README.md index 5113b7b..07ebac6 100644 --- a/nvidia/llama-factory/README.md +++ b/nvidia/llama-factory/README.md @@ -105,9 +105,7 @@ Install the package in editable mode with metrics support for training evaluatio pip install -e ".[metrics]" ``` -## Step 5. Configure PyTorch for CUDA 12.9 (if needed) - -*If using standalone Python (skip if using Docker container)* +## Step 5. Configure PyTorch for CUDA 12.9 (skip if using Docker container from Step 2) In a python virtual environment, uninstall existing PyTorch and reinstall with CUDA 12.9 support for ARM64 architecture. diff --git a/nvidia/multi-agent-chatbot/assets/README.md b/nvidia/multi-agent-chatbot/assets/README.md index 26a08b7..acc8b69 100644 --- a/nvidia/multi-agent-chatbot/assets/README.md +++ b/nvidia/multi-agent-chatbot/assets/README.md @@ -119,6 +119,7 @@ From the root directory of the multi-agent-chatbot project, run the following co docker compose -f docker-compose.yml -f docker-compose-models.yml down docker volume rm "$(basename "$PWD")_postgres_data" +sudo sh -c 'sync; echo 3 > /proc/sys/vm/drop_caches' ``` You can optionally run `docker volume prune` to remove all unused volumes at the end of the demo. > **Note**: If you do not execute these commands containers, will continue to run and take up memory. diff --git a/nvidia/multi-agent-chatbot/assets/backend/agent.py b/nvidia/multi-agent-chatbot/assets/backend/agent.py index ef0da1e..34c2ca9 100644 --- a/nvidia/multi-agent-chatbot/assets/backend/agent.py +++ b/nvidia/multi-agent-chatbot/assets/backend/agent.py @@ -474,7 +474,7 @@ class ChatAgent: config = {"configurable": {"thread_id": chat_id}} try: - existing_messages = await self.conversation_store.get_messages(chat_id, limit=10) + existing_messages = await self.conversation_store.get_messages(chat_id, limit=1) base_system_prompt = self.system_prompt if image_data: diff --git a/nvidia/multi-agent-chatbot/assets/backend/prompts.py b/nvidia/multi-agent-chatbot/assets/backend/prompts.py index d4af5b9..458e85f 100644 --- a/nvidia/multi-agent-chatbot/assets/backend/prompts.py +++ b/nvidia/multi-agent-chatbot/assets/backend/prompts.py @@ -19,7 +19,7 @@ from typing import Dict SUPERVISOR_AGENT_STR = """ -You are a supervisor agent whose role is to be a helpful planner that can use tools to answer questions. DO NOT WRITE CODE YOURSELF, ALWAYS USE THE TOOLS. +You are a supervisor agent whose role is to be a helpful planner that can use tools to answer questions. Please be concise and to the point. {% if tools %} IMPORTANT: You have access to these tools and you MUST use them when applicable and use tool response in your final answer: diff --git a/nvidia/multi-agent-chatbot/assets/docker-compose-models.yml b/nvidia/multi-agent-chatbot/assets/docker-compose-models.yml index 93fe52b..7b5d567 100644 --- a/nvidia/multi-agent-chatbot/assets/docker-compose-models.yml +++ b/nvidia/multi-agent-chatbot/assets/docker-compose-models.yml @@ -122,9 +122,9 @@ services: - "--host" - "0.0.0.0" - "-n" - - "4096" + - "65536" - "--n-gpu-layers" - - "999" + - "70" - "--jinja" deepseek-coder: diff --git a/nvidia/multi-agent-chatbot/assets/frontend/src/components/QuerySection.tsx b/nvidia/multi-agent-chatbot/assets/frontend/src/components/QuerySection.tsx index 155dbbb..86024cc 100644 --- a/nvidia/multi-agent-chatbot/assets/frontend/src/components/QuerySection.tsx +++ b/nvidia/multi-agent-chatbot/assets/frontend/src/components/QuerySection.tsx @@ -15,7 +15,7 @@ # limitations under the License. */ import type React from "react"; -import { useRef, useEffect, useState } from "react"; +import { useRef, useEffect, useState, useCallback } from "react"; import styles from "@/styles/QuerySection.module.css"; import ReactMarkdown from 'react-markdown'; // NEW import remarkGfm from 'remark-gfm'; // NEW @@ -188,22 +188,16 @@ export default function QuerySection({ const chatContainerRef = useRef(null); const [showButtons, setShowButtons] = useState(false); const [showWelcome, setShowWelcome] = useState(true); - const [inferenceStats, setInferenceStats] = useState({ - tokensReceived: 0, - startTime: Date.now(), - tokensPerSecond: 0 - }); const [selectedSources, setSelectedSources] = useState([]); const wsRef = useRef(null); - const [uploadedImage, setUploadedImage] = useState(null); - const [imagePreview, setImagePreview] = useState(null); - const [isDragging, setIsDragging] = useState(false); const [toolOutput, setToolOutput] = useState(""); const [graphStatus, setGraphStatus] = useState(""); const [isPinnedToolOutputVisible, setPinnedToolOutputVisible] = useState(false); const [isToolContentVisible, setIsToolContentVisible] = useState(false); const [fadeIn, setFadeIn] = useState(false); const firstTokenReceived = useRef(false); + const hasAssistantContent = useRef(false); + const fadeTimeoutRef = useRef(null); useEffect(() => { const timer = setTimeout(() => { @@ -212,15 +206,6 @@ export default function QuerySection({ return () => clearTimeout(timer); }, []); - useEffect(() => { - if (!isStreaming) { - setInferenceStats(prev => ({ - ...prev, - tokensReceived: 0, - startTime: 0 - })); - } - }, [isStreaming]); useEffect(() => { const fetchSelectedSources = async () => { @@ -276,9 +261,8 @@ export default function QuerySection({ case "token": { if (!text) break; if (!firstTokenReceived.current) { - console.log('TTFT: ', new Date().toISOString()); firstTokenReceived.current = true; - setIsStreaming(false); + hasAssistantContent.current = true; } setResponse(prev => { try { @@ -310,9 +294,6 @@ export default function QuerySection({ case "tool_end": case "node_end": { console.log(type, msg.data); - if (msg.data === 'generate') { - console.log('generate complete. time: ', new Date().toISOString()); - } setGraphStatus(""); break; } @@ -342,7 +323,6 @@ export default function QuerySection({ return () => { if (wsRef.current) { wsRef.current.close(); - setIsStreaming(false); } }; }, [currentChatId]); @@ -361,70 +341,90 @@ export default function QuerySection({ useEffect(() => { if (graphStatus) { setPinnedToolOutputVisible(true); - } else if (isPinnedToolOutputVisible) { + // Trigger fade-in on next tick + if (fadeTimeoutRef.current) { + clearTimeout(fadeTimeoutRef.current); + } + setFadeIn(false); + fadeTimeoutRef.current = setTimeout(() => setFadeIn(true), 10); + } else { // Delay hiding to allow fade-out + setFadeIn(false); const timeout = setTimeout(() => { setPinnedToolOutputVisible(false); }, 800); // match CSS transition duration - return () => clearTimeout(timeout); + return () => { + clearTimeout(timeout); + if (fadeTimeoutRef.current) { + clearTimeout(fadeTimeoutRef.current); + } + }; } - }, [graphStatus, isPinnedToolOutputVisible]); - - // Replace the effect for fade logic with this minimal version - useEffect(() => { - if (isPinnedToolOutputVisible && graphStatus) { - setFadeIn(false); - const t = setTimeout(() => setFadeIn(true), 10); // next tick for fade-in - return () => clearTimeout(t); - } else { - setFadeIn(false); - } - }, [isPinnedToolOutputVisible, graphStatus]); - - // Cleanup image preview URL on unmount - useEffect(() => { - return () => { - if (imagePreview) { - URL.revokeObjectURL(imagePreview); - } - }; - }, [imagePreview]); + }, [graphStatus]); const programmaticScroll = useRef(false); const scrollTimeout = useRef(null); + const isUserScrollingRef = useRef(false); + const isNearBottomRef = useRef(true); - - const handleDragEnter = (e: React.DragEvent) => { e.preventDefault(); e.stopPropagation(); setIsDragging(true); }; - const handleDragLeave = (e: React.DragEvent) => { e.preventDefault(); e.stopPropagation(); setIsDragging(false); }; - const handleDragOver = (e: React.DragEvent) => { e.preventDefault(); e.stopPropagation(); }; - - const handleDrop = async (e: React.DragEvent) => { - e.preventDefault(); - e.stopPropagation(); - setIsDragging(false); - - const files = Array.from(e.dataTransfer.files); - const imageFile = files.find(file => file.type.startsWith('image/')); - - if (imageFile) { - const previewUrl = URL.createObjectURL(imageFile); - setImagePreview(previewUrl); - - const formData = new FormData(); - formData.append('image', imageFile); - formData.append('chat_id', currentChatId || ''); - - try { - const response = await fetch('/api/upload-image', { method: 'POST', body: formData }); - const result = await response.json(); - setUploadedImage(result.image_id); - } catch (error) { - console.error('Error uploading image:', error); - URL.revokeObjectURL(previewUrl); - setImagePreview(null); - } + // Check if user is near the bottom of the chat + const checkScrollPosition = useCallback(() => { + if (chatContainerRef.current) { + const container = chatContainerRef.current; + const threshold = 100; // pixels from bottom + const isNear = container.scrollHeight - container.scrollTop - container.clientHeight < threshold; + isNearBottomRef.current = isNear; } - }; + }, []); + + // Handle scroll events to detect user scrolling + useEffect(() => { + const container = chatContainerRef.current; + if (!container) return; + + let scrollTimer: NodeJS.Timeout; + + const handleScroll = () => { + isUserScrollingRef.current = true; + checkScrollPosition(); + + // Reset user scrolling flag after scroll stops + clearTimeout(scrollTimer); + scrollTimer = setTimeout(() => { + isUserScrollingRef.current = false; + }, 150); + }; + + container.addEventListener('scroll', handleScroll, { passive: true }); + + return () => { + container.removeEventListener('scroll', handleScroll); + clearTimeout(scrollTimer); + }; + }, [checkScrollPosition]); + + // Auto-scroll to bottom when response changes + useEffect(() => { + // Only scroll if we have assistant content and user hasn't manually scrolled away + if (!hasAssistantContent.current || isUserScrollingRef.current || !isNearBottomRef.current) { + return; + } + + const scrollToBottom = () => { + if (messagesEndRef.current) { + messagesEndRef.current.scrollIntoView({ + behavior: 'smooth', + block: 'end' + }); + } + + if (chatContainerRef.current) { + chatContainerRef.current.scrollTop = chatContainerRef.current.scrollHeight; + } + }; + + scrollToBottom(); + }, [response]); const handleQuerySubmit = async (e: React.FormEvent) => { e.preventDefault(); @@ -434,13 +434,11 @@ export default function QuerySection({ setQuery(""); setIsStreaming(true); firstTokenReceived.current = false; + hasAssistantContent.current = false; try { - console.log('sending uploaded image: ', uploadedImage, ' with query: ', currentQuery) - console.log('current time: ', new Date().toISOString()); wsRef.current.send(JSON.stringify({ - message: currentQuery, - image_id: uploadedImage + message: currentQuery })); setResponse(prev => { @@ -455,14 +453,6 @@ export default function QuerySection({ return prev + `\n\nHuman: ${currentQuery}\n\nAssistant: `; } }); - - // NEW CODE - if (imagePreview) { - URL.revokeObjectURL(imagePreview); - } - setUploadedImage(null); - setImagePreview(null); - // NEW CODE } catch (error) { console.error("Error sending message:", error); setIsStreaming(false); @@ -581,36 +571,7 @@ export default function QuerySection({
- {/* NEW CODE - Image preview moved to the left of inputWrapper */} - {imagePreview && ( -
- Image preview - -
- )} - {/* NEW CODE */} -
+