Skip to content

Add GPU support and enhance monitoring features for jobs #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
stogiannidis wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Add GPU support and enhance monitoring features for jobs #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
19ea10a
Add GPU support to job data generation and monitoring features
stogiannidis Jan 21, 2026
380a339
Enhance GPU resource management and monitoring features in job and po…
stogiannidis Jan 21, 2026
2ca1565
Merge branch 'features/gpu_logs' into develop
stogiannidis Jan 21, 2026
df228df
Refine GPU error handling and enhance log viewer with auto-refresh fe…
stogiannidis Jan 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 87 additions & 5 deletions mock_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,90 +16,105 @@ def _generate_jobs_data(now):
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, hours=2),
"image": "alice/pytorch:2.1",
"gpu": 2,
},
{
"name": "hyperparameter-tuning-job",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, hours=5, minutes=15),
"image": "david/optuna:3.4",
"gpu": 1,
},
{
"name": "image-preprocessing-batch",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, minutes=40),
"image": "frank/opencv:4.8",
"gpu": 0,
},
{
"name": "distributed-training-resnet",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, hours=7, minutes=30),
"image": "bob/horovod:0.28",
"gpu": 4,
},
{
"name": "feature-extraction-pipeline",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, hours=2, minutes=30),
"image": "eve/sklearn:1.3",
"gpu": 0,
},
{
"name": "model-serving-warmup",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, minutes=15),
"image": "carol/triton:23.10",
"gpu": 1,
},
{
"name": "batch-prediction-service",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, hours=3),
"image": "david/tensorflow:2.14",
"gpu": 2,
},
{
"name": "recommendation-engine-train",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, minutes=30),
"image": "frank/lightgbm:4.1",
"gpu": 0,
},
{
"name": "speech-recognition-train",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, hours=3, minutes=45),
"image": "iris/whisper:large-v3",
"gpu": 2,
},
{
"name": "time-series-forecasting",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, hours=5),
"image": "jack/prophet:1.1",
"gpu": 0,
},
{
"name": "semantic-search-indexing",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, hours=7, minutes=15),
"image": "leo/elasticsearch:8.11",
"gpu": 0,
},
{
"name": "document-embedding-job",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, hours=9, minutes=30),
"image": "nancy/sentence-transformers:2.2",
"gpu": 1,
},
{
"name": "reinforcement-learning-agent",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, hours=10, minutes=45),
"image": "peter/stable-baselines3:2.2",
"gpu": 2,
},
{
"name": "anomaly-detection-pipeline",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, hours=13, minutes=15),
"image": "rachel/isolation-forest:1.3",
"gpu": 0,
},
{
"name": "clickstream-analytics",
"active": 1, "succeeded": 0, "failed": 0,
"startTime": _time_ago(now, hours=15),
"image": "tina/flink:1.18",
"gpu": 0,
},
# Completed jobs
{
Expand All @@ -108,62 +123,71 @@ def _generate_jobs_data(now):
"startTime": _time_ago(now, hours=3, minutes=30),
"completionTime": _time_ago(now, hours=2, minutes=45),
"image": "charlie/fastapi:0.104",
"gpu": 1,
},
{
"name": "model-evaluation-suite",
"active": 0, "succeeded": 1, "failed": 0,
"startTime": _time_ago(now, hours=6, minutes=30),
"completionTime": _time_ago(now, hours=6),
"image": "emily/mlflow:2.8",
"gpu": 0,
},
{
"name": "nlp-sentiment-analysis",
"active": 0, "succeeded": 1, "failed": 0,
"startTime": _time_ago(now, hours=9),
"completionTime": _time_ago(now, hours=8, minutes=15),
"image": "henry/transformers:4.35",
"gpu": 1,
},
{
"name": "database-backup-export",
"active": 0, "succeeded": 1, "failed": 0,
"startTime": _time_ago(now, hours=11, minutes=30),
"completionTime": _time_ago(now, hours=11),
"image": "ivan/pgdump:16",
"gpu": 0,
},
{
"name": "log-aggregation-batch",
"active": 0, "succeeded": 1, "failed": 0,
"startTime": _time_ago(now, hours=13, minutes=30),
"completionTime": _time_ago(now, hours=12, minutes=38),
"image": "judy/logstash:8.11",
"gpu": 0,
},
{
"name": "text-classification-job",
"active": 0, "succeeded": 1, "failed": 0,
"startTime": _time_ago(now, hours=1, minutes=30),
"completionTime": _time_ago(now, minutes=48),
"image": "grace/bert-base:1.0",
"gpu": 1,
},
{
"name": "object-detection-yolo",
"active": 0, "succeeded": 1, "failed": 0,
"startTime": _time_ago(now, hours=6, minutes=30),
"completionTime": _time_ago(now, hours=5, minutes=5),
"image": "karen/yolov8:2.0",
"gpu": 2,
},
{
"name": "ab-testing-analysis",
"active": 0, "succeeded": 1, "failed": 0,
"startTime": _time_ago(now, hours=10, minutes=30),
"completionTime": _time_ago(now, hours=10, minutes=12),
"image": "oscar/scipy:1.11",
"gpu": 0,
},
{
"name": "data-lake-sync",
"active": 0, "succeeded": 1, "failed": 0,
"startTime": _time_ago(now, hours=12),
"completionTime": _time_ago(now, hours=11, minutes=8),
"image": "quinn/delta-lake:3.0",
"gpu": 0,
},
# Failed jobs
{
Expand All @@ -172,41 +196,47 @@ def _generate_jobs_data(now):
"startTime": _time_ago(now, hours=4, minutes=30),
"completionTime": _time_ago(now, hours=4, minutes=15),
"image": "emily/spark:3.5",
"gpu": 0,
},
{
"name": "video-encoding-job",
"active": 0, "succeeded": 0, "failed": 1,
"startTime": _time_ago(now, hours=10, minutes=30),
"completionTime": _time_ago(now, hours=10, minutes=25),
"image": "grace/ffmpeg:6.0",
"gpu": 1,
},
{
"name": "etl-customer-data",
"active": 0, "succeeded": 0, "failed": 1,
"startTime": _time_ago(now, hours=8, minutes=30),
"completionTime": _time_ago(now, hours=8, minutes=27),
"image": "ivan/airflow:2.7",
"gpu": 0,
},
{
"name": "fraud-detection-model",
"active": 0, "succeeded": 0, "failed": 1,
"startTime": _time_ago(now, hours=2, minutes=15),
"completionTime": _time_ago(now, hours=2, minutes=10),
"image": "henry/catboost:1.2",
"gpu": 1,
},
{
"name": "graph-neural-network",
"active": 0, "succeeded": 0, "failed": 1,
"startTime": _time_ago(now, hours=8),
"completionTime": _time_ago(now, hours=7, minutes=55),
"image": "maria/pytorch-geometric:2.4",
"gpu": 2,
},
{
"name": "multilingual-translation",
"active": 0, "succeeded": 0, "failed": 1,
"startTime": _time_ago(now, hours=14, minutes=30),
"completionTime": _time_ago(now, hours=14, minutes=22),
"image": "steve/marian-mt:3.1",
"gpu": 1,
},
]

Expand All @@ -216,6 +246,43 @@ def _generate_jobs_data(now):
def _build_jobs_items(jobs_data):
jobs_items = []
for job_info in jobs_data:
# Build resources with GPU if specified
resources = {}
node_selector = {}
gpu_count = job_info.get("gpu", 0)

if gpu_count > 0:
resources = {
"requests": {
"cpu": "20",
"memory": "128Gi",
"nvidia.com/gpu": str(gpu_count)
},
"limits": {
"cpu": "20",
"memory": "128Gi",
"nvidia.com/gpu": str(gpu_count)
}
}
# Assign GPU type based on job requirements (like real jobs do)
if gpu_count >= 4:
node_selector = {"nvidia.com/gpu.product": "NVIDIA-H100-80GB-HBM3"}
elif gpu_count >= 2:
node_selector = {"nvidia.com/gpu.product": "NVIDIA-A100-SXM4-80GB"}
else:
node_selector = {"nvidia.com/gpu.product": "NVIDIA-A100-SXM4-40GB"}

pod_spec = {
"containers": [
{
"image": job_info["image"],
"resources": resources
}
]
}
if node_selector:
pod_spec["nodeSelector"] = node_selector

job = {
"metadata": {"name": job_info["name"]},
"status": {
Expand All @@ -227,11 +294,7 @@ def _build_jobs_items(jobs_data):
"spec": {
"completions": 1,
"template": {
"spec": {
"containers": [
{"image": job_info["image"]}
]
}
"spec": pod_spec
}
}
}
Expand All @@ -251,6 +314,9 @@ def _generate_pod_suffix():

def _generate_pods_items(jobs_data):
pods_items = []
# GPU node assignment based on job GPU requirements
gpu_nodes = ['gpu-node-01', 'gpu-node-02', 'gpu-node-03']

for job_info in jobs_data:
# ~60% get 1 pod, ~30% get 2 pods, ~10% get 3 pods
rand = random.random()
Expand All @@ -262,12 +328,28 @@ def _generate_pods_items(jobs_data):
phase = "Succeeded"
else:
phase = "Failed"

# Assign node based on GPU requirements
gpu_req = job_info.get("gpu", 0)
if gpu_req > 0:
# Assign to a GPU node (prefer A100 for larger jobs)
if gpu_req >= 4:
node_name = 'gpu-node-01' # A100
elif gpu_req >= 2:
node_name = random.choice(['gpu-node-01', 'gpu-node-03']) # A100 or V100
Comment on lines +335 to +339
Copy link

Copilot AI Jan 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's another inconsistency in the mock data node assignments. In _generate_pods_items, the comment on line 337 says "A100" for gpu-node-01, but the get_gpu_info mock data shows gpu-node-01 has H100-80GB GPUs. Similarly, line 339's comment mentions "A100 or V100" but gpu-node-03 is defined as having A100-80GB in get_gpu_info. These comments should be updated to match the actual mock GPU node configuration to avoid confusion.

Suggested change
# Assign to a GPU node (prefer A100 for larger jobs)
if gpu_req >= 4:
node_name = 'gpu-node-01' # A100
elif gpu_req >= 2:
node_name = random.choice(['gpu-node-01', 'gpu-node-03']) # A100 or V100
# Assign to a GPU node (prefer H100 for larger jobs)
if gpu_req >= 4:
node_name = 'gpu-node-01' # H100-80GB
elif gpu_req >= 2:
node_name = random.choice(['gpu-node-01', 'gpu-node-03']) # H100-80GB or A100-80GB

Copilot uses AI. Check for mistakes.
else:
node_name = random.choice(gpu_nodes)
else:
node_name = f"cpu-node-{random.randint(1, 5):02d}"

for _ in range(num_pods):
pods_items.append({
"metadata": {
"name": f"{job_info['name']}-{_generate_pod_suffix()}"
},
"spec": {
"nodeName": node_name
},
"status": {
"phase": phase
}
Expand Down
Loading
Loading