mirror of
https://github.com/NVIDIA/dgx-spark-playbooks.git
synced 2026-04-28 20:53:53 +00:00
3632 lines
1.4 MiB
Plaintext
3632 lines
1.4 MiB
Plaintext
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "31e8ca53",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"# Copyright Notice\n",
|
||
|
|
"\n",
|
||
|
|
"```\n",
|
||
|
|
"SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n",
|
||
|
|
"SPDX-License-Identifier: Apache-2.0\n",
|
||
|
|
"\n",
|
||
|
|
"Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||
|
|
"you may not use this file except in compliance with the License.\n",
|
||
|
|
"You may obtain a copy of the License at\n",
|
||
|
|
"\n",
|
||
|
|
"http://www.apache.org/licenses/LICENSE-2.0\n",
|
||
|
|
"\n",
|
||
|
|
"Unless required by applicable law or agreed to in writing, software\n",
|
||
|
|
"distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
||
|
|
"WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||
|
|
"See the License for the specific language governing permissions and\n",
|
||
|
|
"limitations under the License.\n",
|
||
|
|
"```\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "ae293c8d",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"# VLM-Finetuning for Large Scale Data Analysis"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "0c471d1d",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### 📓 Notebook Overview \n",
|
||
|
|
"In this notebook, we show how to train a VLM to genreate structured metadata about videos. The use case we target is using a VLM to analyze driving videos to generate json-formatted video descriptions and metadata like risky maneuvers to understand dangerous driving patterns. \n",
|
||
|
|
"\n",
|
||
|
|
"This is just one example, this workflow can be generalized to any large scale video data analysis task where it's helpful to have structured metadata and automated video analysis."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "6d02f831",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Intitialize Imports"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "d06618bc",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"/usr/local/lib/python3.12/dist-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
||
|
|
" from .autonotebook import tqdm as notebook_tqdm\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"Appended utils/ to path\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"import logging\n",
|
||
|
|
"\n",
|
||
|
|
"logging.basicConfig(level=logging.INFO, format=\"%(asctime)s - %(levelname)s - %(message)s\")\n",
|
||
|
|
"logger = logging.getLogger(__name__) \n",
|
||
|
|
"\n",
|
||
|
|
"import os\n",
|
||
|
|
"\n",
|
||
|
|
"os.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\" \n",
|
||
|
|
"os.environ[\"TORCH_USE_CUDA_DSA\"] = \"1\"\n",
|
||
|
|
"\n",
|
||
|
|
"import sys\n",
|
||
|
|
"import json\n",
|
||
|
|
"import random\n",
|
||
|
|
"import pathlib\n",
|
||
|
|
"\n",
|
||
|
|
"import torch\n",
|
||
|
|
"import numpy as np\n",
|
||
|
|
"from PIL import Image\n",
|
||
|
|
"from decord import VideoReader\n",
|
||
|
|
"import torch.nn.functional as F\n",
|
||
|
|
"from datasets import load_dataset\n",
|
||
|
|
"from transformers.utils import hub\n",
|
||
|
|
"from tensorboard import program\n",
|
||
|
|
"import torchvision.transforms as T\n",
|
||
|
|
"from IPython.display import display\n",
|
||
|
|
"\n",
|
||
|
|
"from trl import SFTTrainer, SFTConfig\n",
|
||
|
|
"from peft import LoraConfig, get_peft_model\n",
|
||
|
|
"from transformers.trainer_pt_utils import LabelSmoother\n",
|
||
|
|
"from transformers.trainer_utils import get_last_checkpoint\n",
|
||
|
|
"from transformers import AutoModelForCausalLM, AutoTokenizer\n",
|
||
|
|
"from torchvision.transforms.functional import InterpolationMode\n",
|
||
|
|
"from transformers import AutoModelForCausalLM, AutoTokenizer"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "f3cc9f01",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"Starting GPU monitor...\n",
|
||
|
|
"Logging every 5 seconds to logs/gpu-util.csv\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n",
|
||
|
|
"[WARNING] NVML Error: Not Supported\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"# Set seed for reproducibility\n",
|
||
|
|
"def set_seed(seed):\n",
|
||
|
|
" random.seed(seed)\n",
|
||
|
|
" np.random.seed(seed)\n",
|
||
|
|
" torch.manual_seed(seed)\n",
|
||
|
|
" if torch.cuda.is_available():\n",
|
||
|
|
" torch.cuda.manual_seed_all(seed)\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"set_seed(42)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "ef437e65",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"# Set some constants\n",
|
||
|
|
"seq_len = 8192\n",
|
||
|
|
"model_name = \"OpenGVLab/InternVL3-8B\"\n",
|
||
|
|
"ignore_token_id = LabelSmoother.ignore_index"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "0bf4800f",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Model Loading"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "a4e2e572",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"`torch_dtype` is deprecated! Use `dtype` instead!\n",
|
||
|
|
"2025-09-18 01:02:24,054 - INFO - vision_select_layer: -1\n",
|
||
|
|
"2025-09-18 01:02:24,055 - INFO - ps_version: v2\n",
|
||
|
|
"2025-09-18 01:02:24,055 - INFO - min_dynamic_patch: 1\n",
|
||
|
|
"2025-09-18 01:02:24,056 - INFO - max_dynamic_patch: 12\n",
|
||
|
|
"2025-09-18 01:02:24,057 - INFO - vision_config is None. Initializing the InternVisionConfig with default values.\n",
|
||
|
|
"2025-09-18 01:02:24,058 - INFO - llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).\n",
|
||
|
|
"2025-09-18 01:02:24,059 - INFO - vision_select_layer: -1\n",
|
||
|
|
"2025-09-18 01:02:24,059 - INFO - ps_version: v1\n",
|
||
|
|
"2025-09-18 01:02:24,060 - INFO - min_dynamic_patch: 1\n",
|
||
|
|
"2025-09-18 01:02:24,060 - INFO - max_dynamic_patch: 6\n",
|
||
|
|
"2025-09-18 01:02:24,183 - INFO - Discovered apex.normalization.FusedRMSNorm - will use it instead of InternRMSNorm\n",
|
||
|
|
"2025-09-18 01:02:24,187 - INFO - num_image_token: 256\n",
|
||
|
|
"2025-09-18 01:02:24,188 - INFO - ps_version: v2\n",
|
||
|
|
"2025-09-18 01:02:24,374 - INFO - We will use 90% of the memory on device 0 for storing the model, and 10% for the buffer to avoid OOM. You can set `max_memory` in to a higher value to use more memory (at your own risk).\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"trainable params: 4,399,104 || all params: 634,097,024 || trainable%: 0.6938\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"tokenizer = AutoTokenizer.from_pretrained(\n",
|
||
|
|
" model_name, \n",
|
||
|
|
" trust_remote_code=True,\n",
|
||
|
|
" use_fast=True\n",
|
||
|
|
")\n",
|
||
|
|
"tokenizer.padding_side = \"right\"\n",
|
||
|
|
"\n",
|
||
|
|
"model = AutoModelForCausalLM.from_pretrained(\n",
|
||
|
|
" model_name,\n",
|
||
|
|
" torch_dtype=torch.bfloat16,\n",
|
||
|
|
" device_map=\"auto\",\n",
|
||
|
|
" trust_remote_code=True,\n",
|
||
|
|
" use_flash_attn=True,\n",
|
||
|
|
")\n",
|
||
|
|
"\n",
|
||
|
|
"# Load the processor\n",
|
||
|
|
"processor = AutoProcessor.from_pretrained(\n",
|
||
|
|
" model_name,\n",
|
||
|
|
" trust_remote_code=True,\n",
|
||
|
|
")"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "b2c1476a",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Data Processing"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "d3458cd1",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"**Action Required**: Please update the `dataset_path` with a path to your local dataset."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "05d6cd17",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"true\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"dataset_path = \"/lustre/fsw/portfolios/general/users/lramesh/internvl_inference/nexar_scap_dataset_v2\"\n",
|
||
|
|
"if os.path.exists(dataset_path):\n",
|
||
|
|
" print('true')"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "34fad55c",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"# Load conversation template\n",
|
||
|
|
"model_dir = hub.cached_file(model_name, \"conversation.py\", trust_remote_code=True)\n",
|
||
|
|
"sys.path.append(os.path.dirname(model_dir))\n",
|
||
|
|
"\n",
|
||
|
|
"from conversation import get_conv_template"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "f98733c2",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"def get_jsonl_data(sample): \n",
|
||
|
|
" \"\"\" Data processing helper: Define the labels based on the desired format we hope to teach \n",
|
||
|
|
" the model to generate \n",
|
||
|
|
" \"\"\"\n",
|
||
|
|
" answer_dict = {\n",
|
||
|
|
" \"caption\": sample['caption'],\n",
|
||
|
|
" \"event_type\": sample['event_type'],\n",
|
||
|
|
" \"rule_violations\": sample['rule_violations'],\n",
|
||
|
|
" \"intended_action\": sample['intended_action'],\n",
|
||
|
|
" \"traffic_density\": sample['traffic_density'],\n",
|
||
|
|
" \"scene\": sample['scene'],\n",
|
||
|
|
" \"visibility\": sample['visibility'],\n",
|
||
|
|
" }\n",
|
||
|
|
"\n",
|
||
|
|
" return json.dumps(answer_dict, ensure_ascii=False) # create a single line, valid JSON"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "84d2f75f",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Load the Dataset"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "bfb85f7c",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"Generating train split: 4513 examples [00:00, 490220.75 examples/s]\n",
|
||
|
|
"Map: 100%|██████████| 4513/4513 [00:00<00:00, 16999.53 examples/s]\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"# Load the dataset\n",
|
||
|
|
"dataset = load_dataset(dataset_path)\n",
|
||
|
|
"dataset = dataset['train']\n",
|
||
|
|
"dataset = dataset.map(lambda ex: {\"labels\": get_jsonl_data(ex)})\n",
|
||
|
|
"ds_splits = dataset.train_test_split(test_size=0.01, seed=42)\n",
|
||
|
|
"train_dataset, val_dataset = ds_splits['train'], ds_splits['test']"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 10,
|
||
|
|
"id": "bca23383",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"Dataset({\n",
|
||
|
|
" features: ['video', 'caption', 'event_type', 'cause_of_risk', 'presence_of_rule_violations', 'intended_driving_action', 'traffic_density', 'driving_setting', 'time_of_day', 'light_conditions', 'weather', 'scene', 'chunk_metadata', 'original_metadata', 'labels'],\n",
|
||
|
|
" num_rows: 4513\n",
|
||
|
|
"})"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"execution_count": 10,
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "execute_result"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"dataset"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 11,
|
||
|
|
"id": "6b98dbe9",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"text/plain": [
|
||
|
|
"Dataset({\n",
|
||
|
|
" features: ['video', 'caption', 'event_type', 'cause_of_risk', 'presence_of_rule_violations', 'intended_driving_action', 'traffic_density', 'driving_setting', 'time_of_day', 'light_conditions', 'weather', 'scene', 'chunk_metadata', 'original_metadata', 'labels'],\n",
|
||
|
|
" num_rows: 4061\n",
|
||
|
|
"})"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"execution_count": 11,
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "execute_result"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"train_dataset"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "9299e759",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Data Visualization"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 13,
|
||
|
|
"id": "8a95631c",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"def get_video_path(dataset_path, sample_path):\n",
|
||
|
|
" \"\"\" Dataset speciic helper function -- this appends the sample path to the root path to create the full video path \"\"\"\n",
|
||
|
|
" root_dir = dataset_path.split('/')[:-1]\n",
|
||
|
|
" video_path = '/'.join(root_dir) + '/' + sample_path\n",
|
||
|
|
" return video_path"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 14,
|
||
|
|
"id": "dcb4dfba",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"min_frames, max_frames = 8, 32\n",
|
||
|
|
"\n",
|
||
|
|
"# load frames from a video\n",
|
||
|
|
"def load_video(video_path):\n",
|
||
|
|
" video = VideoReader(video_path, num_threads=1)\n",
|
||
|
|
"\n",
|
||
|
|
" # sample a random number of equally-spaced frames from the video\n",
|
||
|
|
" frame_indices = np.linspace(\n",
|
||
|
|
" 0,\n",
|
||
|
|
" len(video) - 1,\n",
|
||
|
|
" random.randint(min_frames, max_frames),\n",
|
||
|
|
" dtype=int\n",
|
||
|
|
" )\n",
|
||
|
|
" frames = video.get_batch(frame_indices).asnumpy()\n",
|
||
|
|
" return [Image.fromarray(frames[i]) for i in range(frames.shape[0])]"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 15,
|
||
|
|
"id": "d388155a",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCALQBQADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD1eim0V6Z5IlFNopkhTaKbTKCiiimSFFMooAWimUUwH0ym06gAptFFMQlFFFAx9MoooAfRTKKAH0UyigB9FMp9MY6im0UAOoptFABTqKKkYU+mUUgH06m0UgMDxtqMmneEbryf+Pi4220H+85x/LNbGnWP9naXa2f3/JiVN3diByT9awPEH+neK/Dek/wJI17L9EHy5/HNdVUAFPplFAD6KZRQA+imUUAPooooGFFFFADqbRRSAdRTaKBDqbTqKYEVFFFABRRRVAFFFFADaKKKACiiigAooooAKKKbQA6im0UwEooopkhRRRQA2iiikAUUU6mA2nUUUAFFFFABT6ZT6ACimU+gYUUUUgCnU2nUhBRRTqAG0UUUFBRRRQAUUUUAFNp1NoJCiiigY2iiimSFFFFABRRRQAUUUUAFFFFABRRRQAlFFFMkSnUUUigooooAKKKKACiiigBKKKKYxaKKKQhKKKfTGMop9OoAbTqKKkAop1FADadRRQA2in0VJQyn0U6gBtFFFIAooooAp2k9tJe3Efmf6V96SPdk7exx71aqrbxQf2hezRx/vnwskn97HbP51coAKdTaloASiin0gG1wWtW8mheI0mj3/Zbj8l9a7+srxDpf9r6PND/y2T5ovqKkoZaSRzxfu5N6SL9ayru0ktw8E/8AwGqPhLUZP+PGT5LiHK+nFb+oxRz/ALz+OP8AlXOawJ9Fu5LvTv3/APx8W/7uX8O9Xq5OK7/s7UIbz/li+Ip/6H8K66uiBgRUU+neXWgEVFNoqwCm0UUyRtFFMpiHUtMrQ06L908n9/5f8aic+U1hDmM+inSReXL5f+cdqjqoEz90KKKbVEC0UU2gkdRTaKAHUU2igB1FNooAdRTaKAH0U2igodRTaKAFooooGFPplFAE1FMopAOp9MopAFPplPjoKOc07/TviLrE/wDz42sVsv1Ylmrp65XwT+8i1vUZP+XrU5f++VwB/WuoqACnVHTqYDqKbRQA6im0UAOoptFSBLRTKKAH0UUygB9FFFAwoplPoEMoooqwCiiikAU2nU2gAooooAKKKKACim0UAOoptFMAooooAKKZT6ZIUUUUAFFFFADKdTadQAUUUUAFFFFABRRRQAU+mU+gAooopDHUUUUhhRRRQAUUUUAFFFFABRRRQSFNp1NoAKKKKYwooplIBaKKKokKKKbSAdRRRTAKKKKACiiigAooooAKKKKQBRRRQAUUUUAFFFFABRRRQAlPooplBRRRSAKdTadSEFFFFADqKbTqAG0UUVIx9FFFMYUUU6pAKbRTKYD6KKKYEfmf6b5H+zuqekPkwSpcv/uSduvQ/nT6yLCiiiqIEoptNoAdRTadQM4XxDB/wj/iO31SH/j3uPvf7Ld66X9zcRRzR/cdan13S/7X0eaz/jdf3f8AvDpXHeFb7zPO0u5/4+Ifu/hWczSBrX1j5W+GT7laXhy++0WX2OT/AF1v8v8AvL2NVpfO+zfvPvpWdb3f9nXsN9/wCdf9k96zHM7anU3/AKaf36dWpBnUU2iukyFoplFMQlFNopkj4o/Ml8utz/V/u/7lYf8Ayx/25PlX+v8Ah+dbFpJ5kX/XP5PyHWuKvM9DDFXVLf8Aded/crN/y3+H610VYkkXlTeX/cy0n0/x/qayhVLqwIPL/wA+2P8AJ/KmVL/00/56f5/n/Koq7aRyVQp9RUVrMxNX7DbR/wCsuX/T/CmfZLP/AJ/H/T/Cm6v/AMfKf7v9TWfXPDmlHmudU+WMti3cWn2fZ+83o/3fWo4IpLiXy4//AK1T3f8AyD7L8P8A0Gp7WCT+y/3f35aj2svZ+ZXso83yIvs9hH/rLl9/+fam3Fp+68yH97D+q07+yJ/9j9f8Ks2NjNby/vPuP9aXP53HyeRn2kcdxL5ckjp/d24/wqKWLyJfLk/yPWmyfupX/wBhq1bfy9Ui/effT71ac/L73RmcIc3u9UVPskf2Lz5t/wD0z24+b8xVe3jkk/dx/wCfrUt1d+fL/sJ92p7T9xp01z/H/kf1qeeX37D5Pe5ew77JZxf66f56j+w+Z/qZN6bhu/vIPeqNWbT93cp/vfzqp+0jHcXPT7D7q3+zzeX/AOhVJ9kj8lJLmbyt/wBP8KbqP/H7J+H8qkvv9Vb/AO6f6VHPLTzL5I6+RHJHDGP3M2/8v8KgplKK6YHPMWiiiqMQp9MooLJabL+6tppP7kTN+QNFU9Uk/wCJPe/9cn/9BNSUZvgOLyvBVh/013y/99MTXR1j+GP3fhXSY/8Ap3WtWkA+imUUwH0Uyn0gCnU2ikA6iiigAp1NooAdRRTaAHUU2nUAFFFNoAdRRRQAU2iigAooptMB1FMopkj6bRRSKCnU2igAooplADqKbRTJCiiigAoop9ABRRRQAUUUUhCUUUUxhRRRQAUUUUAFPplPoAKKKKkYUUUUwCiiigB9FMooJHUU2mUxj6KZRQA+n1FRSEFFFFMAooptIB1FNp1ABRTadTAKKKKACim06gAooooAKKKKACiiikAUUUUAFFFFABRRRQA6nUUUDCiiikAUUUUAMp9FFSMKdRRQIbRRRQMdRRRTELRRTKRQ+imUUAPplFPoAbTqKKQHPeII5NUlh0mCCX7we6bcVTyj/CSOSSQOB6V0Nv8A88/L/wBV8q/SmQf8hG48uT995Y3LU/meZczfu/k/ve/tWZqPooptWZBTKfRQAU6iipAK888YWn9i+I4dXh/1M3+s/wB7v+deh1ma7Yx61o9xZ/x7fl+ooAqR3H2i2S8mk+R1Hvx6VnXdpH88f30/xrI8JXf7p9Onj/fQyfxe3UY/Cuh1H/S4vM8v50/lmsOQ6Cz4bvvMs/sc3+ut/l/4B2Na/mVxnmf2dqMN59Fl/wB0/wCFdRWsDnCimU2usgWm0UUyRKdTaZUlQJ//ANhf8f1/M1LY3H2e5/2P4vTcf84+gqn5v3P88+tNrl9kdvtYnT1Svrf/AJ5/x48xv9kc1NY3Hn21QapP5dt5f9/+Vc0ObmN+f3TIlk8z/wBl/DpTabSV60DzJzH0R/66mUppz6kw+I0Na/4+U/3f6ms6pbu7kupfMk2f8BqCs6XwmtWfvGrff8g6y/4D/wCg0W/+l6c8H8afMtZ8l3JLbQwybP3WP0GKiikkj/eR1h7L9353Nfax5vkPk8yKpoIJ5/3ke/8Az6VP/a8n/LSFH/SobjUZ54vL+5/u0/e/lsHu/wAzIoopLiXy460p7j
|
||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAABQAAAALQCAIAAABAH0oBAAEAAElEQVR4Aez9eZAdR37nCcaL490v35F3JhKJiwB4gkeRVcVSVakOnaVuk0ZqU7e6p3vUNjbWbTPba7v79/Y/a7a2a/v3ms3umO10q1vqmZ7RUVJJpa6DpTrE4l0kAYIgQAJIIO/jHfnuF8fbz88jAYLICBYfAZKJojuSj/E8PDzcf+4Rz7/+/R2Jby25xijJHY5SmrKuGXlBEASR+e5w1BtEVhObGXPb2PJxJwLfjzwVGFZ0fhBdPjFidy0zun4/pv648pGN/BgyTTN6Pty1WwfR89mMkZthRI+LZUXL2TCiByyuX71+I51OZ7KZfr+fSCQCPxgOh3/0R39UrVW/9MUvnTp9Kp/Ld7tdMovFYqvVSmfSkaL4pMYxEdmau5dpWtF3CGLmcyJ2HEdrU1z9vh89vvG1x82T+CtGORP3nhmljk+07NAZ6fbuMPp3YaRK3qdw3Li/zyV35VT8++euVG9YH/V7Na6ZiZj3bez7M66i6Hxz1Mcxupq7lhtEv66Mu/WcesFH+z65a4L4hCryY+Qf15xEzDrto34e49ozav6o68PhqPL5iJ8vy4peX5l36f0QJ8+49ZjhR7+v4uqJy49djyU+2t+vufn0xbd21tbXksnkeGV8fLzCgrJWrbHCjGxq3O9C3PwfdXnlDL3I+77wwotrq6udbpd2ZjKZubnZiYkJ1sDlimGyDB4ObdsuZLK24zR2quvr6267JyVTKRrs9gftTqfbalP++Be/OD097fvB7u5u0k6blul7CRbU/d5A3deWz+G7mMKLWTdGNpLMd6+MK6HztQS0BD60BAC9XCu4NxiaCZP31MbGxsrKysW3LpYr5empKXkRFAq2Y7c7bV4BH/pG+kItAS0BLQEtAS0BLQEtAS2BX0gJrK70XM/t9Xr1ep2DwcDlv2KpeNA6Ozk5mc3lQK3Ly8ubm5u0ttVqQ/M0Gg0y2+12t9NxXQ+4m8vlSqUS7XddF8Q7cAXZ2pbFwphdkuvXr3c6HegodjT8IGA5zQnPj0bdH0IIGgB/CKHpS7QEPqgE2LKiKE8sKg88w9lM4q2Lb126dIlHfWpqKpMxeBek08lsJuu5nuOMRpd90EbocloCWgJaAloCWgJaAloCWgL3rARAvPAlM9PTMzMzMMAAwm6vdwB7Mzszw/oW2ArcbTWbjXqjXq95ns8SF9aXlbDn+71eF4rY8zx6QfI8t9MR1pfupFQCBr/99tubGxvkQA5BHXNASfiku9VlDYDvliR1PVoCERLgcSXX53EPfHnIfeONc2801tae+uxTY2NjjYbL3li/7zpJh0c+To89ol6dpSWgJaAloCWgJaAloCWgJfDpkEAy6QygfIulI4uHM1kHTpV1o2MrTeCDJAFQ+qFDhxYOL2DZZ9lWq92C+/V9D3VoKF8+AbcukHcwgBmC7AXfJhImnDB/CTMhONkGKTtbW1srq6ugaOlmMgn05S9kle5KdzUAviti1JVoCURLALVnTvCe4pMHe31t553L7xi2/dWvfJWcTreDhTAKLSSe8F73IG7mRXdM52oJaAloCWgJaAloCWgJaAl8LBKAIL165UqjUUdZsNMeABhLxYx98BQHAb3YJy8sLGD6C2sNjq3X6ig5s9D1fR9wC7oFA6PpDBgGJAOYOQDewxJhLYgspQzA2HHQoMZskBxWyKBltCkhlu+WsO9aRXerQboeLYFfPAnwVOM1gQf7/PnzG+sbc0ePLC4uspXFIz0zO0N/2602VsK8Gn7x+q57pCWgJaAloCWgJaAloCWgJXAnEgBMKhI0GAygVWQBCVrc2W7cSZ0fxbVQPklH3HQBbgHtYgPcRhW6hcNXVrmoQANiKQOA5x8r4XK5XKmUUYqkR+SDcvkEH8/OzfW63fW19U67zSX4ymEhDUt8t9qsAfDdkqSuR0sgQgLAWnLlcVcP/Llz57BzeOD+B1D5IPEwO3aCZ54yPPAoQkdUobO0BLQEtAS0BLQEtAS0BLQEPsUSYCU5Pz/fbDZffOmljc1NVpVQqgfQDBhMy4IWN9WQwJVyGT1HcuCBoXPRZ0aLGxgPIYwKNF6vAMEVSeOgYEpyIZgZ22B6d+TIIlzxTnWHaykGUUyBbrdzt6bAgdMdv1sd0/VoCRwECfAw0wyArq/iLV27fs1wPUwjUBFJp232xdjrAgCj/gFLLK6w7p6Du4PQfd0GLQEtAS0BLQEtAS0BLQEtgTuUgO95uWz2Jz/+8fPPv/D0F56ulCv5fG52ZrrVvmuY8A5bGF4OvoX0qZQLR48eWV9fIxPoXqtViXUC7OUrONYhpqVly4HtZLNZUX72PaBvv9vDaQ4BwADAoGIKEBSp3mjMDgblMtGezM5aN53K3pV22nHxeONqV+rZcSej8mP8dcXeVzkNiqooOm/UuI4jhokyGMjIGzO6kfmjtieykvfJHNVP0qjl3+fWI50C8kWWjx33yNKjZ+I6ffSLIq6I00bGH3tEaQIdxcyT0J6BZ57HGyUQ/k3Nzz/15FMAY/RBgMHUxiPP24Gc90G/n9Q43i15RgqNzFGfl1HLx8a7i5kn+GmIa6rOfx8JxM1/Y9S4vuIz7h5IcfPqoDV91N+vT6r9rHWib333HH5G169zf6ElcK88pwdtEOLWt3HtZAETfWrE93nseyC69pFz49alo1YU185Q42/U2j54+ZOnpi5d3EYHOJvLHjt6DPRrO4latYn17AevhJKx42WMhnfMYfR6G/Vs20ni8xWI/tBDD29v72C6PD4x0WztiHdowkN7HsxvqVQmYBILYKx8IYF4WtGRru7ssFrOZ3N8rq6s3H///T/72avPPvtsKpkdDo9g+Tw3Ozfoh/Gc1d1vCX4d3Rrc0MYAP80AjzRtdGEtgdEkgIIHBG86wy6X+fLLF9HlmBifCOOejVaRLq0loCWgJaAloCWgJaAloCXwqZTA1pZs0KNCWCqXDi8eNs3E7m4H4iRpjAaAPwbhsScO/YMmdDaTmZgYR4cZD1igXG7NqSRK0Z4PIxTmZ+1UyDnB9+byeVSjKQZO5hN4jHo0wJiQwmUQc2UKXehR2w+WjrwkZtsmsqzO1BLQEhhRAuw4huQYdr54wMJ449ixY9j6j1iNLq4loCWgJaAloCWgJaAloCXwKZUAxGkmm5mbnX3ooYfyubznB+BDLGMPmjjQZ4Smhv5BZY/l7uLiEZheAv/iCAsz4O3tbeAums90B1SMgiSalXxyVaGQD22G4dhJ4GfMhLkW0uj69etgYNuyKTZqf4nGEv03akW6vJaAlsAHlwC2DTyuxDdqd4YXL13E8dWp06fi9U8+eMW6pJaAloCWgJaAloCWgJaAlsCnQgIoCcORZjJZUOVNIpR4Qgew84BbMDCr33wht3j48NzcLNi1g0tnldrtNm3mLM6uQnqW4qBluobj6GwuZ9k2+SF7RJfzihYGABMUCeQ8an+pKjJpFehRJanLawmMJgH2sYgAzL7XtWvX2LQ7euSobUXrY4xWry6tJaAloCWgJaAloCWgJaAl8CmQAKDXCkx0nokeZCYSySRkiomDqEw6faB6D9okuhGWwLTKtg0cPM/PzV+dWLIau1C+1WqVwEiJYEizLUNChKIu7Xk+CsmAWwwGcfSFRxwqASE7jgMJDHWECTFXvXXxIs6lR+0si/DIS6JzI4vqTC0BLYFRJcDTjg0wVhBLV5dq1RqWG+IDQO87jSpHXV5LQEtAS0BLQEtAS0BL4NMqAVaSBD1CkRjTWdhUXHZCmYKED5o8Egnx6gpfzZ/nEQbFYN17/PjxUFu7Vqutrq6CZvHRCJ+NQy+YXuhfKGN6R8Ak1sykkP6lHgyAsQeGQ6ab15auQSaN2l8oqMg/DYBHlaQuryUwggR4epNJdDmMC29d4DKioqG70u34I1Shi2oJaAloCWgJaAloCWgJaAl8iiWQSuH1aheUiHktZCn8iuPYYOCDJhKAL/wtbUM9m8YO3KBYKp44cWJiYgI/WABa3D6LJbAnQUDRcA6jb9CpDl6eBwNi/0L5AoDBvd1OBztnihX
|
||
|
|
"text/plain": [
|
||
|
|
"<PIL.Image.Image image mode=RGB size=1280x720>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "display_data"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"video_path = get_video_path(dataset_path, train_dataset[0]['video'])\n",
|
||
|
|
"display(load_video(video_path)[0])"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "2b21a463",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Data Processing\n",
|
||
|
|
"Functions taken from [InternVL3 Documentation](https://internvl.readthedocs.io/en/latest/internvl3.0/quick_start.html#inference-with-transformers)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 16,
|
||
|
|
"id": "8f8e4c46",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"# preprocessing code directly adopted HF model card\n",
|
||
|
|
"IMAGENET_MEAN = (0.485, 0.456, 0.406)\n",
|
||
|
|
"IMAGENET_STD = (0.229, 0.224, 0.225)\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"def build_transform(input_size):\n",
|
||
|
|
" MEAN, STD = IMAGENET_MEAN, IMAGENET_STD\n",
|
||
|
|
" transform = T.Compose([\n",
|
||
|
|
" T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),\n",
|
||
|
|
" T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),\n",
|
||
|
|
" T.ToTensor(),\n",
|
||
|
|
" T.Normalize(mean=MEAN, std=STD)\n",
|
||
|
|
" ])\n",
|
||
|
|
" return transform\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):\n",
|
||
|
|
" best_ratio_diff = float('inf')\n",
|
||
|
|
" best_ratio = (1, 1)\n",
|
||
|
|
" area = width * height\n",
|
||
|
|
" for ratio in target_ratios:\n",
|
||
|
|
" target_aspect_ratio = ratio[0] / ratio[1]\n",
|
||
|
|
" ratio_diff = abs(aspect_ratio - target_aspect_ratio)\n",
|
||
|
|
" if ratio_diff < best_ratio_diff:\n",
|
||
|
|
" best_ratio_diff = ratio_diff\n",
|
||
|
|
" best_ratio = ratio\n",
|
||
|
|
" elif ratio_diff == best_ratio_diff:\n",
|
||
|
|
" if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:\n",
|
||
|
|
" best_ratio = ratio\n",
|
||
|
|
" return best_ratio\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbnail=False):\n",
|
||
|
|
" orig_width, orig_height = image.size\n",
|
||
|
|
" aspect_ratio = orig_width / orig_height\n",
|
||
|
|
"\n",
|
||
|
|
" # calculate the existing image aspect ratio\n",
|
||
|
|
" target_ratios = set(\n",
|
||
|
|
" (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if\n",
|
||
|
|
" i * j <= max_num and i * j >= min_num)\n",
|
||
|
|
" target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])\n",
|
||
|
|
"\n",
|
||
|
|
" # find the closest aspect ratio to the target\n",
|
||
|
|
" target_aspect_ratio = find_closest_aspect_ratio(\n",
|
||
|
|
" aspect_ratio, target_ratios, orig_width, orig_height, image_size)\n",
|
||
|
|
"\n",
|
||
|
|
" # calculate the target width and height\n",
|
||
|
|
" target_width = image_size * target_aspect_ratio[0]\n",
|
||
|
|
" target_height = image_size * target_aspect_ratio[1]\n",
|
||
|
|
" blocks = target_aspect_ratio[0] * target_aspect_ratio[1]\n",
|
||
|
|
"\n",
|
||
|
|
" # resize the image\n",
|
||
|
|
" resized_img = image.resize((target_width, target_height))\n",
|
||
|
|
" processed_images = []\n",
|
||
|
|
" for i in range(blocks):\n",
|
||
|
|
" box = (\n",
|
||
|
|
" (i % (target_width // image_size)) * image_size,\n",
|
||
|
|
" (i // (target_width // image_size)) * image_size,\n",
|
||
|
|
" ((i % (target_width // image_size)) + 1) * image_size,\n",
|
||
|
|
" ((i // (target_width // image_size)) + 1) * image_size\n",
|
||
|
|
" )\n",
|
||
|
|
" # split the image\n",
|
||
|
|
" split_img = resized_img.crop(box)\n",
|
||
|
|
" processed_images.append(split_img)\n",
|
||
|
|
" assert len(processed_images) == blocks\n",
|
||
|
|
" if use_thumbnail and len(processed_images) != 1:\n",
|
||
|
|
" thumbnail_img = image.resize((image_size, image_size))\n",
|
||
|
|
" processed_images.append(thumbnail_img)\n",
|
||
|
|
" return processed_images\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"# build the transform and get number of tokens per image (per tile technically)\n",
|
||
|
|
"image_size = model.config.force_image_size\n",
|
||
|
|
"transform = build_transform(input_size=image_size)\n",
|
||
|
|
"num_image_tokens = int((image_size // model.config.vision_config.patch_size) ** 2 * (model.config.downsample_ratio ** 2))"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "6097f018",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Define user prompt"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "e24f2a4a",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"user_prompt = \"\"\"You are a vision-language assistant analyzing driving videos. You will receive a 5-second video clip of a specific scene. \n",
|
||
|
|
"\n",
|
||
|
|
"---\n",
|
||
|
|
"\n",
|
||
|
|
"### Task 1: Dense Caption\n",
|
||
|
|
"Generate a 2 sentence caption describing:\n",
|
||
|
|
"- Ego vehicle behavior\n",
|
||
|
|
"- Interactions with other vehicles or pedestrians\n",
|
||
|
|
"\n",
|
||
|
|
"Focus on **what happens**, **when**, and **who/what is involved**, using only visible information and metadata.\n",
|
||
|
|
"\n",
|
||
|
|
"---\n",
|
||
|
|
"\n",
|
||
|
|
"### Task 2: Structured JSON\n",
|
||
|
|
"Generate the caption from the perspective of the ego vehicle in a structured JSON object with:\n",
|
||
|
|
"\n",
|
||
|
|
"- `\"caption\"`: from Task 1 \n",
|
||
|
|
"- `\"event_type\"`: \"collision\" | \"near_miss\" | \"no_incident\" \n",
|
||
|
|
"- `\"rule_violations\"`: choose relevant items from [\"speeding\", \"failure_to_yield\", \"ignoring_traffic_signs\"] \n",
|
||
|
|
"- `\"intended_action\"`: \"turn_left\" | \"turn_right\" | \"change_lanes\" \n",
|
||
|
|
"- `\"traffic_density\"`: \"low\" | \"high\" \n",
|
||
|
|
"- `\"visibility\"`: \"good\" | \"bad\" \n",
|
||
|
|
"- `\"scene\"`: \"Urban\" | \"Sub-urban\" | \"Rural\" | \"Highway\"\n",
|
||
|
|
"\n",
|
||
|
|
"**Rules:**\n",
|
||
|
|
"1. Use only visible info and metadata. \n",
|
||
|
|
"2. Do not invent details. \n",
|
||
|
|
"3. Include all fields; enum values must match allowed options. \n",
|
||
|
|
"4. Output a single valid JSON object—no extra text or markdown. \n",
|
||
|
|
"\"\"\""
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "9e82e25f",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"{\"caption\": \"The video shows a stationary vehicle at an urban intersection under a bridge. There are no significant movements or interactions with other vehicles or pedestrians, and the scene remains calm with clear weather conditions.\", \"event_type\": \"no_incident\", \"cause_of_risk\": [], \"presence_of_rule_violations\": [], \"intended_driving_action\": \"enter_road\", \"traffic_density\": \"low\", \"driving_setting\": \"intersection\", \"time_of_day\": \"day\", \"light_conditions\": \"Normal\", \"weather\": \"Clear\", \"scene\": \"Urban\"}\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"# View a sample label to confirm it's in the format we want\n",
|
||
|
|
"sample = train_dataset[0]\n",
|
||
|
|
"\n",
|
||
|
|
"answer_dict = {\n",
|
||
|
|
" \"caption\": sample['caption'],\n",
|
||
|
|
" \"event_type\": sample['event_type'],\n",
|
||
|
|
" \"rule_violations\": sample['rule_violations'],\n",
|
||
|
|
" \"intended_action\": sample['intended_action'],\n",
|
||
|
|
" \"traffic_density\": sample['traffic_density'],\n",
|
||
|
|
" \"scene\": sample['scene'],\n",
|
||
|
|
" \"visibility\": sample['visibility'],\n",
|
||
|
|
"}\n",
|
||
|
|
"answer_jsonl = json.dumps(answer_dict, ensure_ascii=False)\n",
|
||
|
|
"\n",
|
||
|
|
"print(answer_jsonl)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "24b3e453",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Custom Data Preprocessing\n",
|
||
|
|
"\n",
|
||
|
|
"This novel data tokenization function takes in a batch of samples and tokenizes them according to what InternVL3 source code expects, returning a processed batch of features with input IDs, labels, attention masks, etc."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "59243cf6",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"# Collate function for the dataset that does on-the-fly preprocessing and batching\n",
|
||
|
|
"def collate_fn(samples, tokenizer, transform, seq_len, num_image_tokens, get_conv_template, ignore_token_id, load_video, get_video_path, dataset_path):\n",
|
||
|
|
" input_ids_batch, labels_batch, attention_mask_batch, position_ids_batch, pixel_values_batch, image_flags_batch = [], [], [], [], [], []\n",
|
||
|
|
" for sample in samples:\n",
|
||
|
|
" # load the video frames\n",
|
||
|
|
" video_frames = load_video(get_video_path(dataset_path, sample['video']))\n",
|
||
|
|
" num_frames = len(video_frames)\n",
|
||
|
|
"\n",
|
||
|
|
" # preprocess the video frames\n",
|
||
|
|
" pixel_values = [transform(frame) for frame in video_frames]\n",
|
||
|
|
" pixel_values = torch.stack(pixel_values)\n",
|
||
|
|
" num_tiles = pixel_values.size(0)\n",
|
||
|
|
"\n",
|
||
|
|
" # prepend special video tokens to the user message\n",
|
||
|
|
" video_tokens = '\\n'.join(['Frame-{}: <image>'.format(i + 1) for i in range(num_frames)])\n",
|
||
|
|
"\n",
|
||
|
|
" # setup conversation\n",
|
||
|
|
" conv_template = get_conv_template(\"internvl2_5\")\n",
|
||
|
|
"\n",
|
||
|
|
" system_instruction = user_prompt\n",
|
||
|
|
" answer = get_jsonl_data(sample)\n",
|
||
|
|
"\n",
|
||
|
|
" conv_template.append_message(conv_template.roles[0], f'{video_tokens}\\n{system_instruction}')\n",
|
||
|
|
" conv_template.append_message(conv_template.roles[1], answer)\n",
|
||
|
|
"\n",
|
||
|
|
" # replace image tokens with context tokens\n",
|
||
|
|
" prompt = conv_template.get_prompt()\n",
|
||
|
|
" prompt = prompt.replace(\"<image>\", f\"<img>{'<IMG_CONTEXT>' * num_image_tokens}</img>\")\n",
|
||
|
|
"\n",
|
||
|
|
" # create a list of messages\n",
|
||
|
|
" messages = [f\"<|im_start|>{message}\" for message in prompt.split(\"<|im_start|>\")[1: ]]\n",
|
||
|
|
"\n",
|
||
|
|
" # tokenize the prompt (we manually truncate and pad the sequence)\n",
|
||
|
|
" input_ids = tokenizer(\n",
|
||
|
|
" messages,\n",
|
||
|
|
" return_tensors=\"np\",\n",
|
||
|
|
" padding=False,\n",
|
||
|
|
" max_length=seq_len,\n",
|
||
|
|
" truncation=False,\n",
|
||
|
|
" ).input_ids\n",
|
||
|
|
"\n",
|
||
|
|
" # create targets by masking out system and user messages\n",
|
||
|
|
" # since we only want to compute loss for the assistant message\n",
|
||
|
|
" targets = []\n",
|
||
|
|
" num_ignore_ids = tokenizer('<|im_start|>assistant\\n', return_tensors='np').input_ids[0].shape[0]\n",
|
||
|
|
" for idx, input_id in enumerate(input_ids):\n",
|
||
|
|
" if idx != 2:\n",
|
||
|
|
" targets.append(np.full(input_id.shape, ignore_token_id))\n",
|
||
|
|
" else:\n",
|
||
|
|
" target = input_id.copy()\n",
|
||
|
|
" target[: num_ignore_ids] = ignore_token_id\n",
|
||
|
|
" target[-1: ] = ignore_token_id\n",
|
||
|
|
" targets.append(target)\n",
|
||
|
|
"\n",
|
||
|
|
" # prepare the input_ids and targets\n",
|
||
|
|
" input_ids = torch.tensor(np.concatenate(input_ids))[: seq_len]\n",
|
||
|
|
" targets = torch.tensor(np.concatenate(targets))[: seq_len]\n",
|
||
|
|
"\n",
|
||
|
|
" # pad the input_ids and targets to the sequence length\n",
|
||
|
|
" pad_len = seq_len - input_ids.shape[0]\n",
|
||
|
|
" input_ids = F.pad(input_ids, (0, pad_len), value=tokenizer.pad_token_id)\n",
|
||
|
|
" targets = F.pad(targets, (0, pad_len), value=ignore_token_id)\n",
|
||
|
|
"\n",
|
||
|
|
" # generate attention mask to filter out padding tokens\n",
|
||
|
|
" attention_mask = input_ids.ne(tokenizer.pad_token_id)\n",
|
||
|
|
"\n",
|
||
|
|
" position_ids = attention_mask.long().cumsum(-1) - 1\n",
|
||
|
|
" position_ids.masked_fill_(attention_mask == 0, 1)\n",
|
||
|
|
"\n",
|
||
|
|
" input_ids_batch.append(input_ids)\n",
|
||
|
|
" labels_batch.append(targets)\n",
|
||
|
|
" attention_mask_batch.append(attention_mask)\n",
|
||
|
|
" position_ids_batch.append(position_ids)\n",
|
||
|
|
" pixel_values_batch.append(pixel_values)\n",
|
||
|
|
" image_flags_batch.append(torch.tensor([1] * num_tiles, dtype=torch.long))\n",
|
||
|
|
"\n",
|
||
|
|
" batch = {\n",
|
||
|
|
" \"input_ids\": torch.stack(input_ids_batch),\n",
|
||
|
|
" \"labels\": torch.stack(labels_batch),\n",
|
||
|
|
" \"attention_mask\": torch.stack(attention_mask_batch),\n",
|
||
|
|
" \"position_ids\": torch.stack(position_ids_batch),\n",
|
||
|
|
" \"pixel_values\": torch.cat(pixel_values_batch),\n",
|
||
|
|
" \"image_flags\": torch.cat(image_flags_batch)\n",
|
||
|
|
" }\n",
|
||
|
|
"\n",
|
||
|
|
" return batch"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "6a8912c4",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"## Training"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "36c55eb2",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Set Model Config Params for Training"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "59766bcf",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"model.img_context_token_id = tokenizer.convert_tokens_to_ids(\"<IMG_CONTEXT>\")\n",
|
||
|
|
"\n",
|
||
|
|
"model.train()\n",
|
||
|
|
"\n",
|
||
|
|
"model.language_model.config.use_cache = False\n",
|
||
|
|
"model.vision_model.gradient_checkpointing = True\n",
|
||
|
|
"model.vision_model.encoder.gradient_checkpointing = True\n",
|
||
|
|
"model.language_model._set_gradient_checkpointing()"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "b6c2dec6",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Define Training Params"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"id": "686f3cfd",
|
||
|
|
"metadata": {
|
||
|
|
"lines_to_next_cell": 2
|
||
|
|
},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"save_dir = \"eval-run-1\" # feel free to adjust\n",
|
||
|
|
"\n",
|
||
|
|
"trainer = SFTTrainer(\n",
|
||
|
|
" model=model,\n",
|
||
|
|
" data_collator=collate_fn,\n",
|
||
|
|
" train_dataset=train_dataset,\n",
|
||
|
|
" eval_dataset=val_dataset,\n",
|
||
|
|
" processing_class=processor,\n",
|
||
|
|
" args=SFTConfig(\n",
|
||
|
|
" num_train_epochs=30,\n",
|
||
|
|
" per_device_train_batch_size=1,\n",
|
||
|
|
" per_device_eval_batch_size=1,\n",
|
||
|
|
" eval_steps=250,\n",
|
||
|
|
" do_eval=True,\n",
|
||
|
|
" warmup_ratio=0.03,\n",
|
||
|
|
" lr_scheduler_type=\"cosine\",\n",
|
||
|
|
" eval_strategy=\"steps\",\n",
|
||
|
|
" label_names=[\"labels\"],\n",
|
||
|
|
" dataloader_num_workers=4,\n",
|
||
|
|
" gradient_accumulation_steps=4,\n",
|
||
|
|
" dataloader_persistent_workers=True,\n",
|
||
|
|
" learning_rate=2e-5,\n",
|
||
|
|
" weight_decay=0.05,\n",
|
||
|
|
" logging_steps=10,\n",
|
||
|
|
" logging_dir=\"logs\",\n",
|
||
|
|
" save_strategy=\"steps\",\n",
|
||
|
|
" save_steps=100,\n",
|
||
|
|
" output_dir=save_dir,\n",
|
||
|
|
" save_total_limit=2,\n",
|
||
|
|
" optim=\"adamw_torch\",\n",
|
||
|
|
" bf16=True,\n",
|
||
|
|
" remove_unused_columns=False,\n",
|
||
|
|
" report_to=\"wandb\",\n",
|
||
|
|
" dataset_kwargs = {\"skip_prepare_dataset\": True},\n",
|
||
|
|
"\n",
|
||
|
|
" )\n",
|
||
|
|
")"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "7c364912",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"### Train the model\n",
|
||
|
|
"**Note:** Remove the `resume_from_checkpoint` parameter of `trainer.train()` if you don't want to resume training from a checkpoint."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 23,
|
||
|
|
"id": "7ab9fa74",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"The tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: {'eos_token_id': 151645, 'pad_token_id': 151643}.\n",
|
||
|
|
"2025-09-18 01:02:39,337 - INFO - vision_config is None. Initializing the InternVisionConfig with default values.\n",
|
||
|
|
"2025-09-18 01:02:39,339 - INFO - llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).\n",
|
||
|
|
"2025-09-18 01:02:39,339 - INFO - vision_select_layer: -1\n",
|
||
|
|
"2025-09-18 01:02:39,340 - INFO - ps_version: v1\n",
|
||
|
|
"2025-09-18 01:02:39,340 - INFO - min_dynamic_patch: 1\n",
|
||
|
|
"2025-09-18 01:02:39,340 - INFO - max_dynamic_patch: 6\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"Resuming from None\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py:893: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.5 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
|
||
|
|
" return fn(*args, **kwargs)\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"text/html": [
|
||
|
|
"\n",
|
||
|
|
" <div>\n",
|
||
|
|
" \n",
|
||
|
|
" <progress value='46' max='30480' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
|
||
|
|
" [ 46/30480 35:50 < 413:15:07, 0.02 it/s, Epoch 0.04/30]\n",
|
||
|
|
" </div>\n",
|
||
|
|
" <table border=\"1\" class=\"dataframe\">\n",
|
||
|
|
" <thead>\n",
|
||
|
|
" <tr style=\"text-align: left;\">\n",
|
||
|
|
" <th>Step</th>\n",
|
||
|
|
" <th>Training Loss</th>\n",
|
||
|
|
" <th>Validation Loss</th>\n",
|
||
|
|
" </tr>\n",
|
||
|
|
" </thead>\n",
|
||
|
|
" <tbody>\n",
|
||
|
|
" </tbody>\n",
|
||
|
|
"</table><p>"
|
||
|
|
],
|
||
|
|
"text/plain": [
|
||
|
|
"<IPython.core.display.HTML object>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "display_data"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"2025-09-18 01:27:08,429 - INFO - vision_config is None. Initializing the InternVisionConfig with default values.\n",
|
||
|
|
"2025-09-18 01:27:08,429 - INFO - llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).\n",
|
||
|
|
"2025-09-18 01:27:08,430 - INFO - vision_select_layer: -1\n",
|
||
|
|
"2025-09-18 01:27:08,430 - INFO - ps_version: v1\n",
|
||
|
|
"2025-09-18 01:27:08,430 - INFO - min_dynamic_patch: 1\n",
|
||
|
|
"2025-09-18 01:27:08,430 - INFO - max_dynamic_patch: 6\n",
|
||
|
|
"2025-09-18 01:27:08,431 - INFO - vision_config is None. Initializing the InternVisionConfig with default values.\n",
|
||
|
|
"2025-09-18 01:27:08,431 - INFO - llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).\n",
|
||
|
|
"2025-09-18 01:27:08,431 - INFO - vision_select_layer: -1\n",
|
||
|
|
"2025-09-18 01:27:08,431 - INFO - ps_version: v1\n",
|
||
|
|
"2025-09-18 01:27:08,431 - INFO - min_dynamic_patch: 1\n",
|
||
|
|
"2025-09-18 01:27:08,431 - INFO - max_dynamic_patch: 6\n",
|
||
|
|
"2025-09-18 01:27:08,433 - INFO - vision_config is None. Initializing the InternVisionConfig with default values.\n",
|
||
|
|
"2025-09-18 01:27:08,434 - INFO - llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).\n",
|
||
|
|
"2025-09-18 01:27:08,434 - INFO - vision_select_layer: -1\n",
|
||
|
|
"2025-09-18 01:27:08,434 - INFO - ps_version: v1\n",
|
||
|
|
"2025-09-18 01:27:08,434 - INFO - min_dynamic_patch: 1\n",
|
||
|
|
"2025-09-18 01:27:08,435 - INFO - max_dynamic_patch: 6\n",
|
||
|
|
"/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py:893: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.5 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
|
||
|
|
" return fn(*args, **kwargs)\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ename": "KeyboardInterrupt",
|
||
|
|
"evalue": "",
|
||
|
|
"output_type": "error",
|
||
|
|
"traceback": [
|
||
|
|
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
||
|
|
"\u001b[31mKeyboardInterrupt\u001b[39m Traceback (most recent call last)",
|
||
|
|
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[23]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m last_ckpt = get_last_checkpoint(save_dir)\n\u001b[32m 2\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mResuming from \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mlast_ckpt\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m \u001b[43mtrainer\u001b[49m\u001b[43m.\u001b[49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlast_ckpt\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
|
|
"\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.12/dist-packages/transformers/trainer.py:2328\u001b[39m, in \u001b[36mTrainer.train\u001b[39m\u001b[34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[39m\n\u001b[32m 2326\u001b[39m hf_hub_utils.enable_progress_bars()\n\u001b[32m 2327\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m2328\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minner_training_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 2329\u001b[39m \u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m=\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2330\u001b[39m \u001b[43m \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2331\u001b[39m \u001b[43m \u001b[49m\u001b[43mtrial\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2332\u001b[39m \u001b[43m \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m=\u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2333\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n",
|
||
|
|
"\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.12/dist-packages/transformers/trainer.py:2672\u001b[39m, in \u001b[36mTrainer._inner_training_loop\u001b[39m\u001b[34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[39m\n\u001b[32m 2665\u001b[39m context = (\n\u001b[32m 2666\u001b[39m functools.partial(\u001b[38;5;28mself\u001b[39m.accelerator.no_sync, model=model)\n\u001b[32m 2667\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m i != \u001b[38;5;28mlen\u001b[39m(batch_samples) - \u001b[32m1\u001b[39m\n\u001b[32m 2668\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m.accelerator.distributed_type != DistributedType.DEEPSPEED\n\u001b[32m 2669\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m contextlib.nullcontext\n\u001b[32m 2670\u001b[39m )\n\u001b[32m 2671\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m context():\n\u001b[32m-> \u001b[39m\u001b[32m2672\u001b[39m tr_loss_step = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtraining_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_items_in_batch\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 2674\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[32m 2675\u001b[39m args.logging_nan_inf_filter\n\u001b[32m 2676\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_torch_xla_available()\n\u001b[32m 2677\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m (torch.isnan(tr_loss_step) \u001b[38;5;129;01mor\u001b[39;00m torch.isinf(tr_loss_step))\n\u001b[32m 2678\u001b[39m ):\n\u001b[32m 2679\u001b[39m \u001b[38;5;66;03m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[32m 2680\u001b[39m tr_loss = tr_loss + tr_loss / (\u001b[32m1\u001b[39m + \u001b[38;5;28mself\u001b[39m.state.global_step - \u001b[38;5;28mself\u001b[39m._globalstep_last_logged)\n",
|
||
|
|
"\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.12/dist-packages/trl/trainer/sft_trainer.py:872\u001b[39m, in \u001b[36mSFTTrainer.training_step\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 870\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mtraining_step\u001b[39m(\u001b[38;5;28mself\u001b[39m, *args, **kwargs):\n\u001b[32m 871\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m.maybe_activation_offload_context:\n\u001b[32m--> \u001b[39m\u001b[32m872\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mtraining_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
|
|
"\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.12/dist-packages/transformers/trainer.py:4060\u001b[39m, in \u001b[36mTrainer.training_step\u001b[39m\u001b[34m(***failed resolving arguments***)\u001b[39m\n\u001b[32m 4057\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.accelerator.distributed_type == DistributedType.DEEPSPEED:\n\u001b[32m 4058\u001b[39m kwargs[\u001b[33m\"\u001b[39m\u001b[33mscale_wrt_gas\u001b[39m\u001b[33m\"\u001b[39m] = \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m4060\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43maccelerator\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mloss\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 4062\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m loss.detach()\n",
|
||
|
|
"\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.12/dist-packages/accelerate/accelerator.py:2734\u001b[39m, in \u001b[36mAccelerator.backward\u001b[39m\u001b[34m(self, loss, **kwargs)\u001b[39m\n\u001b[32m 2732\u001b[39m \u001b[38;5;28mself\u001b[39m.lomo_backward(loss, learning_rate)\n\u001b[32m 2733\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m2734\u001b[39m \u001b[43mloss\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
|
|
"\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.12/dist-packages/torch/_tensor.py:648\u001b[39m, in \u001b[36mTensor.backward\u001b[39m\u001b[34m(self, gradient, retain_graph, create_graph, inputs)\u001b[39m\n\u001b[32m 638\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[32m 639\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[32m 640\u001b[39m Tensor.backward,\n\u001b[32m 641\u001b[39m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[32m (...)\u001b[39m\u001b[32m 646\u001b[39m inputs=inputs,\n\u001b[32m 647\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m648\u001b[39m \u001b[43mtorch\u001b[49m\u001b[43m.\u001b[49m\u001b[43mautograd\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 649\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m=\u001b[49m\u001b[43minputs\u001b[49m\n\u001b[32m 650\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
|
|
"\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.12/dist-packages/torch/autograd/__init__.py:354\u001b[39m, in \u001b[36mbackward\u001b[39m\u001b[34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[39m\n\u001b[32m 349\u001b[39m retain_graph = create_graph\n\u001b[32m 351\u001b[39m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[32m 352\u001b[39m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[32m 353\u001b[39m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m354\u001b[39m \u001b[43m_engine_run_backward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 355\u001b[39m \u001b[43m \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 356\u001b[39m \u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 357\u001b[39m \u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 358\u001b[39m \u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 359\u001b[39m \u001b[43m \u001b[49m\u001b[43minputs_tuple\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 360\u001b[39m \u001b[43m \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 361\u001b[39m \u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 362\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
|
|
"\u001b[36mFile \u001b[39m\u001b[32m/usr/local/lib/python3.12/dist-packages/torch/autograd/graph.py:829\u001b[39m, in \u001b[36m_engine_run_backward\u001b[39m\u001b[34m(t_outputs, *args, **kwargs)\u001b[39m\n\u001b[32m 827\u001b[39m unregister_hooks = _register_logging_hooks_on_whole_graph(t_outputs)\n\u001b[32m 828\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m829\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mVariable\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_execution_engine\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[32m 830\u001b[39m \u001b[43m \u001b[49m\u001b[43mt_outputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\n\u001b[32m 831\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[32m 832\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 833\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m attach_logging_hooks:\n",
|
||
|
|
"\u001b[31mKeyboardInterrupt\u001b[39m: "
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"last_ckpt = get_last_checkpoint(save_dir)\n",
|
||
|
|
"print(f\"Resuming from {last_ckpt}\")\n",
|
||
|
|
"trainer.train(resume_from_checkpoint=last_ckpt)"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"jupytext": {
|
||
|
|
"cell_metadata_filter": "-all",
|
||
|
|
"encoding": "# coding: utf-8",
|
||
|
|
"executable": "/usr/bin/env python",
|
||
|
|
"main_language": "python",
|
||
|
|
"notebook_metadata_filter": "-all"
|
||
|
|
},
|
||
|
|
"kernelspec": {
|
||
|
|
"display_name": "Python 3",
|
||
|
|
"language": "python",
|
||
|
|
"name": "python3"
|
||
|
|
},
|
||
|
|
"language_info": {
|
||
|
|
"codemirror_mode": {
|
||
|
|
"name": "ipython",
|
||
|
|
"version": 3
|
||
|
|
},
|
||
|
|
"file_extension": ".py",
|
||
|
|
"mimetype": "text/x-python",
|
||
|
|
"name": "python",
|
||
|
|
"nbconvert_exporter": "python",
|
||
|
|
"pygments_lexer": "ipython3",
|
||
|
|
"version": "3.12.3"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"nbformat": 4,
|
||
|
|
"nbformat_minor": 5
|
||
|
|
}
|