Initial scaffold: LLM Trainer Dashboard
Full-stack app with FastAPI backend (SSH/paramiko, pipeline streaming, GPU stats, xterm.js terminal, Ollama model manager) and React + Tailwind frontend (8 panels: Connection, Documents, Pipeline, QA Pairs, Training, Terminal, Models, Config). Docker Compose included. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
33
.gitignore
vendored
Normal file
33
.gitignore
vendored
Normal file
@@ -0,0 +1,33 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.pyo
|
||||
.env
|
||||
.venv/
|
||||
venv/
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
|
||||
# Node
|
||||
node_modules/
|
||||
frontend/dist/
|
||||
frontend/.vite/
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
desktop.ini
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
|
||||
# Secrets
|
||||
*.pem
|
||||
*.key
|
||||
id_rsa
|
||||
id_ed25519
|
||||
.env.local
|
||||
.env.production
|
||||
12
backend/Dockerfile
Normal file
12
backend/Dockerfile
Normal file
@@ -0,0 +1,12 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"]
|
||||
39
backend/gpu.py
Normal file
39
backend/gpu.py
Normal file
@@ -0,0 +1,39 @@
|
||||
from ssh_client import ssh_manager
|
||||
|
||||
|
||||
def get_gpu_stats() -> dict:
|
||||
"""Query nvidia-smi on the remote host and return parsed GPU info."""
|
||||
try:
|
||||
if not ssh_manager.is_connected():
|
||||
return {"gpus": [], "error": "Not connected"}
|
||||
|
||||
out, err, code = ssh_manager.execute(
|
||||
"nvidia-smi --query-gpu=name,utilization.gpu,memory.used,memory.total,"
|
||||
"temperature.gpu,power.draw --format=csv,noheader,nounits",
|
||||
use_conda=False
|
||||
)
|
||||
|
||||
if code != 0:
|
||||
return {"gpus": [], "error": err.strip() or "nvidia-smi failed"}
|
||||
|
||||
gpus = []
|
||||
for line in out.strip().split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
parts = [p.strip() for p in line.split(",")]
|
||||
if len(parts) >= 5:
|
||||
try:
|
||||
gpus.append({
|
||||
"name": parts[0],
|
||||
"utilization": int(parts[1]),
|
||||
"memory_used": int(parts[2]),
|
||||
"memory_total": int(parts[3]),
|
||||
"temperature": int(parts[4]),
|
||||
"power_draw": float(parts[5]) if len(parts) > 5 else None,
|
||||
})
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
|
||||
return {"gpus": gpus, "error": None}
|
||||
except Exception as e:
|
||||
return {"gpus": [], "error": str(e)}
|
||||
450
backend/main.py
Normal file
450
backend/main.py
Normal file
@@ -0,0 +1,450 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
import yaml
|
||||
from fastapi import FastAPI, File, HTTPException, UploadFile, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from pydantic import BaseModel
|
||||
|
||||
from gpu import get_gpu_stats
|
||||
from pipeline import STAGE_DIRS, CONFIG_PATH, ingest_cmd, create_cmd, curate_cmd, save_as_cmd, train_cmd
|
||||
from ssh_client import ssh_manager
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
app = FastAPI(title="LLM Trainer API", version="1.0.0")
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://192.168.2.47:11434")
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Pydantic models
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
class ConnectRequest(BaseModel):
|
||||
host: str = "192.168.2.47"
|
||||
username: str = "tocmo0nlord"
|
||||
password: Optional[str] = None
|
||||
key_path: Optional[str] = None
|
||||
port: int = 22
|
||||
|
||||
|
||||
class TrainRequest(BaseModel):
|
||||
model_name: str = "llama3.1:8b"
|
||||
dataset_path: str
|
||||
output_dir: str = "/opt/synthetic/output"
|
||||
num_epochs: int = 3
|
||||
batch_size: int = 2
|
||||
learning_rate: float = 2e-4
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Helpers
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _require_ssh():
|
||||
if not ssh_manager.is_connected():
|
||||
raise HTTPException(status_code=503, detail="Not connected to SSH server")
|
||||
|
||||
|
||||
async def _stream_ws(websocket: WebSocket, command: str, use_conda: bool = True):
|
||||
"""Run a remote command and stream output lines over WebSocket."""
|
||||
await websocket.accept()
|
||||
loop = asyncio.get_event_loop()
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
|
||||
def _worker():
|
||||
try:
|
||||
for line in ssh_manager.execute_stream(command, use_conda=use_conda):
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
queue.put({"type": "log", "data": line}), loop
|
||||
)
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
queue.put({"type": "done", "data": "Command completed."}), loop
|
||||
)
|
||||
except Exception as exc:
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
queue.put({"type": "error", "data": str(exc)}), loop
|
||||
)
|
||||
|
||||
threading.Thread(target=_worker, daemon=True).start()
|
||||
|
||||
try:
|
||||
while True:
|
||||
msg = await queue.get()
|
||||
await websocket.send_json(msg)
|
||||
if msg["type"] in ("done", "error"):
|
||||
break
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Connection
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@app.post("/api/connect")
|
||||
async def connect(req: ConnectRequest):
|
||||
try:
|
||||
ssh_manager.connect(
|
||||
host=req.host,
|
||||
username=req.username,
|
||||
password=req.password,
|
||||
key_path=req.key_path,
|
||||
port=req.port,
|
||||
)
|
||||
return {"status": "connected", "host": req.host, "username": req.username}
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc))
|
||||
|
||||
|
||||
@app.post("/api/disconnect")
|
||||
async def disconnect():
|
||||
ssh_manager.disconnect()
|
||||
return {"status": "disconnected"}
|
||||
|
||||
|
||||
@app.get("/api/status")
|
||||
async def status():
|
||||
connected = ssh_manager.is_connected()
|
||||
gpu = get_gpu_stats() if connected else {"gpus": [], "error": "Not connected"}
|
||||
return {
|
||||
"connected": connected,
|
||||
"host": ssh_manager.host if connected else None,
|
||||
"username": ssh_manager.username if connected else None,
|
||||
"gpu": gpu,
|
||||
}
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# GPU
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@app.get("/api/gpu")
|
||||
async def gpu():
|
||||
_require_ssh()
|
||||
return get_gpu_stats()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# File management
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@app.get("/api/files/{stage}")
|
||||
async def list_files(stage: str):
|
||||
if stage not in STAGE_DIRS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown stage: {stage}")
|
||||
_require_ssh()
|
||||
|
||||
out, _, code = ssh_manager.execute(
|
||||
f"ls -la '{STAGE_DIRS[stage]}' 2>/dev/null | tail -n +2", use_conda=False
|
||||
)
|
||||
|
||||
files = []
|
||||
for line in out.strip().split("\n"):
|
||||
if not line.strip() or line.startswith("total"):
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) >= 9 and not parts[0].startswith("d"):
|
||||
files.append({
|
||||
"name": " ".join(parts[8:]),
|
||||
"size": int(parts[4]),
|
||||
"modified": f"{parts[5]} {parts[6]} {parts[7]}",
|
||||
})
|
||||
|
||||
return {"stage": stage, "directory": STAGE_DIRS[stage], "files": files}
|
||||
|
||||
|
||||
@app.delete("/api/files/{stage}/{filename}")
|
||||
async def delete_file(stage: str, filename: str):
|
||||
if stage not in STAGE_DIRS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown stage: {stage}")
|
||||
_require_ssh()
|
||||
|
||||
path = f"{STAGE_DIRS[stage]}/{filename}"
|
||||
_, err, code = ssh_manager.execute(f"rm -f '{path}'", use_conda=False)
|
||||
if code != 0:
|
||||
raise HTTPException(status_code=500, detail=err)
|
||||
return {"deleted": filename}
|
||||
|
||||
|
||||
@app.get("/api/files/{stage}/{filename}/preview")
|
||||
async def preview_file(stage: str, filename: str, lines: int = 120):
|
||||
if stage not in STAGE_DIRS:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown stage: {stage}")
|
||||
_require_ssh()
|
||||
|
||||
path = f"{STAGE_DIRS[stage]}/{filename}"
|
||||
out, err, code = ssh_manager.execute(f"head -n {lines} '{path}'", use_conda=False)
|
||||
if code != 0:
|
||||
raise HTTPException(status_code=500, detail=err)
|
||||
return {"filename": filename, "content": out}
|
||||
|
||||
|
||||
@app.post("/api/upload")
|
||||
async def upload_file(file: UploadFile = File(...)):
|
||||
_require_ssh()
|
||||
|
||||
suffix = Path(file.filename).suffix
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
|
||||
tmp.write(await file.read())
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
remote_path = f"{STAGE_DIRS['input']}/{file.filename}"
|
||||
ssh_manager.upload_file(tmp_path, remote_path)
|
||||
return {"uploaded": file.filename, "remote_path": remote_path}
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Pipeline (WebSocket streaming)
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@app.websocket("/api/pipeline/ingest")
|
||||
async def ws_ingest(websocket: WebSocket, filename: str):
|
||||
if not ssh_manager.is_connected():
|
||||
await websocket.accept()
|
||||
await websocket.send_json({"type": "error", "data": "Not connected"})
|
||||
return
|
||||
cmd = ingest_cmd(f"{STAGE_DIRS['input']}/{filename}")
|
||||
await _stream_ws(websocket, cmd)
|
||||
|
||||
|
||||
@app.websocket("/api/pipeline/create")
|
||||
async def ws_create(websocket: WebSocket, filename: str,
|
||||
num_pairs: int = 50, pair_type: str = "qa"):
|
||||
if not ssh_manager.is_connected():
|
||||
await websocket.accept()
|
||||
await websocket.send_json({"type": "error", "data": "Not connected"})
|
||||
return
|
||||
cmd = create_cmd(f"{STAGE_DIRS['parsed']}/{filename}", num_pairs, pair_type)
|
||||
await _stream_ws(websocket, cmd)
|
||||
|
||||
|
||||
@app.websocket("/api/pipeline/curate")
|
||||
async def ws_curate(websocket: WebSocket, filename: str,
|
||||
output_filename: str, threshold: float = 7.0):
|
||||
if not ssh_manager.is_connected():
|
||||
await websocket.accept()
|
||||
await websocket.send_json({"type": "error", "data": "Not connected"})
|
||||
return
|
||||
cmd = curate_cmd(
|
||||
f"{STAGE_DIRS['generated']}/{filename}",
|
||||
f"{STAGE_DIRS['curated']}/{output_filename}",
|
||||
threshold,
|
||||
)
|
||||
await _stream_ws(websocket, cmd)
|
||||
|
||||
|
||||
@app.websocket("/api/pipeline/save")
|
||||
async def ws_save(websocket: WebSocket, filename: str,
|
||||
output_filename: str, fmt: str = "jsonl"):
|
||||
if not ssh_manager.is_connected():
|
||||
await websocket.accept()
|
||||
await websocket.send_json({"type": "error", "data": "Not connected"})
|
||||
return
|
||||
cmd = save_as_cmd(
|
||||
f"{STAGE_DIRS['curated']}/{filename}",
|
||||
f"{STAGE_DIRS['final']}/{output_filename}",
|
||||
fmt,
|
||||
)
|
||||
await _stream_ws(websocket, cmd)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# QA Pairs viewer
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@app.get("/api/pairs/{filename}")
|
||||
async def get_pairs(filename: str, stage: str = "generated"):
|
||||
_require_ssh()
|
||||
path = f"{STAGE_DIRS.get(stage, STAGE_DIRS['generated'])}/{filename}"
|
||||
out, err, code = ssh_manager.execute(f"cat '{path}'", use_conda=False)
|
||||
if code != 0:
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {filename}")
|
||||
|
||||
pairs = []
|
||||
for line in out.strip().split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
pairs.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
return {"filename": filename, "count": len(pairs), "pairs": pairs}
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Config editor
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@app.get("/api/config")
|
||||
async def get_config():
|
||||
_require_ssh()
|
||||
try:
|
||||
raw = ssh_manager.read_remote_file(CONFIG_PATH)
|
||||
return {"config": yaml.safe_load(raw), "raw": raw}
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc))
|
||||
|
||||
|
||||
@app.put("/api/config")
|
||||
async def update_config(payload: dict):
|
||||
_require_ssh()
|
||||
try:
|
||||
ssh_manager.write_remote_file(CONFIG_PATH, yaml.dump(payload, default_flow_style=False))
|
||||
return {"status": "updated"}
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc))
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Training (WebSocket streaming)
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@app.websocket("/api/train")
|
||||
async def ws_train(
|
||||
websocket: WebSocket,
|
||||
model_name: str = "llama3.1:8b",
|
||||
dataset_path: str = "",
|
||||
output_dir: str = "/opt/synthetic/output",
|
||||
num_epochs: int = 3,
|
||||
batch_size: int = 2,
|
||||
learning_rate: float = 2e-4,
|
||||
):
|
||||
if not ssh_manager.is_connected():
|
||||
await websocket.accept()
|
||||
await websocket.send_json({"type": "error", "data": "Not connected"})
|
||||
return
|
||||
|
||||
cmd = train_cmd(model_name, dataset_path, output_dir, num_epochs, batch_size, learning_rate)
|
||||
await _stream_ws(websocket, cmd)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Interactive terminal (xterm.js ↔ SSH shell)
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@app.websocket("/api/terminal")
|
||||
async def ws_terminal(websocket: WebSocket):
|
||||
await websocket.accept()
|
||||
|
||||
if not ssh_manager.is_connected():
|
||||
await websocket.send_text("\r\nNot connected to SSH server.\r\n")
|
||||
return
|
||||
|
||||
channel = None
|
||||
try:
|
||||
channel = ssh_manager.open_shell_channel()
|
||||
|
||||
async def ssh_to_ws():
|
||||
while True:
|
||||
if channel.recv_ready():
|
||||
data = channel.recv(4096)
|
||||
if not data:
|
||||
break
|
||||
await websocket.send_bytes(data)
|
||||
elif channel.exit_status_ready():
|
||||
break
|
||||
else:
|
||||
await asyncio.sleep(0.02)
|
||||
|
||||
async def ws_to_ssh():
|
||||
try:
|
||||
while True:
|
||||
data = await websocket.receive_bytes()
|
||||
channel.send(data)
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
|
||||
await asyncio.gather(ssh_to_ws(), ws_to_ssh())
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
except Exception as exc:
|
||||
try:
|
||||
await websocket.send_text(f"\r\nError: {exc}\r\n")
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
if channel:
|
||||
try:
|
||||
channel.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Model manager (Ollama)
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@app.get("/api/models")
|
||||
async def list_models():
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
resp = await client.get(f"{OLLAMA_URL}/api/tags")
|
||||
resp.raise_for_status()
|
||||
return {"models": resp.json().get("models", [])}
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc))
|
||||
|
||||
|
||||
@app.websocket("/api/models/pull")
|
||||
async def ws_pull_model(websocket: WebSocket, model_name: str):
|
||||
await websocket.accept()
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=600) as client:
|
||||
async with client.stream(
|
||||
"POST", f"{OLLAMA_URL}/api/pull",
|
||||
json={"name": model_name, "stream": True}
|
||||
) as resp:
|
||||
async for line in resp.aiter_lines():
|
||||
if line.strip():
|
||||
try:
|
||||
await websocket.send_json(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
await websocket.send_json({"status": "success"})
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
except Exception as exc:
|
||||
try:
|
||||
await websocket.send_json({"status": "error", "error": str(exc)})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@app.delete("/api/models/{model_name:path}")
|
||||
async def delete_model(model_name: str):
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.request(
|
||||
"DELETE", f"{OLLAMA_URL}/api/delete",
|
||||
json={"name": model_name}
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return {"deleted": model_name}
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc))
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=8080, reload=True)
|
||||
73
backend/pipeline.py
Normal file
73
backend/pipeline.py
Normal file
@@ -0,0 +1,73 @@
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Pipeline paths & command builders
|
||||
# These match the remote Ubuntu server layout from LLM_TRAINER_APP_SCOPE.md
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
SDK_BIN = (
|
||||
"/home/tocmo0nlord/miniconda3/envs/synthetic-data/bin/synthetic-data-kit"
|
||||
)
|
||||
CONFIG_PATH = "/opt/synthetic/synthetic-data-kit/config.yaml"
|
||||
DATA_BASE = "/opt/synthetic/synthetic-data-kit/data"
|
||||
|
||||
STAGE_DIRS = {
|
||||
"input": f"{DATA_BASE}/input",
|
||||
"parsed": f"{DATA_BASE}/parsed",
|
||||
"generated": f"{DATA_BASE}/generated",
|
||||
"curated": f"{DATA_BASE}/curated",
|
||||
"final": f"{DATA_BASE}/final",
|
||||
}
|
||||
|
||||
TRAIN_SCRIPT = "/opt/synthetic/train.py"
|
||||
OUTPUT_BASE = "/opt/synthetic/output"
|
||||
|
||||
|
||||
def _sdk(subcommand: str, *args) -> str:
|
||||
return f"{SDK_BIN} --config {CONFIG_PATH} {subcommand} {' '.join(args)}"
|
||||
|
||||
|
||||
def ingest_cmd(input_file: str) -> str:
|
||||
return _sdk("ingest", f"'{input_file}'", "-o", STAGE_DIRS["parsed"])
|
||||
|
||||
|
||||
def create_cmd(parsed_file: str, num_pairs: int = 50, pair_type: str = "qa") -> str:
|
||||
return _sdk(
|
||||
"create", f"'{parsed_file}'",
|
||||
"-o", STAGE_DIRS["generated"],
|
||||
"--type", pair_type,
|
||||
"--num-pairs", str(num_pairs),
|
||||
)
|
||||
|
||||
|
||||
def curate_cmd(generated_file: str, output_file: str, threshold: float = 7.0) -> str:
|
||||
return _sdk(
|
||||
"curate", f"'{generated_file}'",
|
||||
"-o", f"'{output_file}'",
|
||||
"--threshold", str(threshold),
|
||||
)
|
||||
|
||||
|
||||
def save_as_cmd(curated_file: str, output_file: str, fmt: str = "jsonl") -> str:
|
||||
return _sdk(
|
||||
"save-as", f"'{curated_file}'",
|
||||
"-f", fmt,
|
||||
"-o", f"'{output_file}'",
|
||||
)
|
||||
|
||||
|
||||
def train_cmd(
|
||||
model_name: str,
|
||||
dataset_path: str,
|
||||
output_dir: str = OUTPUT_BASE,
|
||||
num_epochs: int = 3,
|
||||
batch_size: int = 2,
|
||||
learning_rate: float = 2e-4,
|
||||
) -> str:
|
||||
return (
|
||||
f"python3 {TRAIN_SCRIPT} "
|
||||
f"--model '{model_name}' "
|
||||
f"--dataset '{dataset_path}' "
|
||||
f"--output '{output_dir}' "
|
||||
f"--epochs {num_epochs} "
|
||||
f"--batch-size {batch_size} "
|
||||
f"--lr {learning_rate}"
|
||||
)
|
||||
7
backend/requirements.txt
Normal file
7
backend/requirements.txt
Normal file
@@ -0,0 +1,7 @@
|
||||
fastapi==0.111.0
|
||||
uvicorn[standard]==0.29.0
|
||||
paramiko==3.4.0
|
||||
httpx==0.27.0
|
||||
pyyaml==6.0.1
|
||||
python-multipart==0.0.9
|
||||
websockets==12.0
|
||||
177
backend/ssh_client.py
Normal file
177
backend/ssh_client.py
Normal file
@@ -0,0 +1,177 @@
|
||||
import base64
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
import paramiko
|
||||
|
||||
|
||||
class SSHClient:
|
||||
def __init__(self):
|
||||
self.client: Optional[paramiko.SSHClient] = None
|
||||
self.connected = False
|
||||
self.host = ""
|
||||
self.username = ""
|
||||
self.port = 22
|
||||
self._keepalive_thread: Optional[threading.Thread] = None
|
||||
self._stop_keepalive = threading.Event()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def connect(self, host: str, username: str, password: str = None,
|
||||
key_path: str = None, port: int = 22) -> bool:
|
||||
with self._lock:
|
||||
try:
|
||||
if self.client:
|
||||
self.client.close()
|
||||
|
||||
self.client = paramiko.SSHClient()
|
||||
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
kwargs = {"hostname": host, "port": port, "username": username, "timeout": 10}
|
||||
if key_path:
|
||||
kwargs["key_filename"] = key_path
|
||||
if password:
|
||||
kwargs["password"] = password
|
||||
|
||||
self.client.connect(**kwargs)
|
||||
self.connected = True
|
||||
self.host = host
|
||||
self.username = username
|
||||
self.port = port
|
||||
|
||||
self._stop_keepalive.clear()
|
||||
self._keepalive_thread = threading.Thread(target=self._keepalive_loop, daemon=True)
|
||||
self._keepalive_thread.start()
|
||||
return True
|
||||
except Exception as e:
|
||||
self.connected = False
|
||||
raise e
|
||||
|
||||
def disconnect(self):
|
||||
self._stop_keepalive.set()
|
||||
if self.client:
|
||||
self.client.close()
|
||||
self.connected = False
|
||||
|
||||
def _keepalive_loop(self):
|
||||
while not self._stop_keepalive.wait(30):
|
||||
try:
|
||||
transport = self.client.get_transport()
|
||||
if transport and transport.is_active():
|
||||
transport.send_ignore()
|
||||
else:
|
||||
self.connected = False
|
||||
break
|
||||
except Exception:
|
||||
self.connected = False
|
||||
break
|
||||
|
||||
def execute(self, command: str, use_conda: bool = True) -> tuple:
|
||||
if not self.is_connected():
|
||||
raise Exception("Not connected to SSH server")
|
||||
|
||||
if use_conda:
|
||||
full_cmd = (
|
||||
f"source /home/{self.username}/miniconda3/etc/profile.d/conda.sh && "
|
||||
f"conda activate synthetic-data && {command}"
|
||||
)
|
||||
else:
|
||||
full_cmd = command
|
||||
|
||||
_, stdout, stderr = self.client.exec_command(full_cmd)
|
||||
out = stdout.read().decode("utf-8", errors="replace")
|
||||
err = stderr.read().decode("utf-8", errors="replace")
|
||||
exit_code = stdout.channel.recv_exit_status()
|
||||
return out, err, exit_code
|
||||
|
||||
def execute_stream(self, command: str, use_conda: bool = True):
|
||||
"""Generator that yields output lines from a command."""
|
||||
if not self.is_connected():
|
||||
raise Exception("Not connected to SSH server")
|
||||
|
||||
if use_conda:
|
||||
full_cmd = (
|
||||
f"source /home/{self.username}/miniconda3/etc/profile.d/conda.sh && "
|
||||
f"conda activate synthetic-data && {command}"
|
||||
)
|
||||
else:
|
||||
full_cmd = command
|
||||
|
||||
transport = self.client.get_transport()
|
||||
channel = transport.open_session()
|
||||
channel.get_pty()
|
||||
channel.exec_command(full_cmd)
|
||||
|
||||
buffer = b""
|
||||
while True:
|
||||
if channel.recv_ready():
|
||||
data = channel.recv(4096)
|
||||
if not data:
|
||||
break
|
||||
buffer += data
|
||||
while b"\n" in buffer:
|
||||
line, buffer = buffer.split(b"\n", 1)
|
||||
yield line.decode("utf-8", errors="replace") + "\n"
|
||||
elif channel.exit_status_ready():
|
||||
if buffer:
|
||||
yield buffer.decode("utf-8", errors="replace")
|
||||
break
|
||||
else:
|
||||
time.sleep(0.05)
|
||||
|
||||
channel.close()
|
||||
|
||||
def open_shell_channel(self, term: str = "xterm-256color", width: int = 220, height: int = 50):
|
||||
"""Open an interactive shell channel for the terminal panel."""
|
||||
if not self.is_connected():
|
||||
raise Exception("Not connected to SSH server")
|
||||
|
||||
transport = self.client.get_transport()
|
||||
channel = transport.open_session()
|
||||
channel.get_pty(term=term, width=width, height=height)
|
||||
channel.invoke_shell()
|
||||
|
||||
# Auto-activate conda env
|
||||
activate = (
|
||||
f"source /home/{self.username}/miniconda3/etc/profile.d/conda.sh && "
|
||||
f"conda activate synthetic-data\n"
|
||||
)
|
||||
channel.send(activate)
|
||||
return channel
|
||||
|
||||
def upload_file(self, local_path: str, remote_path: str):
|
||||
if not self.is_connected():
|
||||
raise Exception("Not connected to SSH server")
|
||||
sftp = self.client.open_sftp()
|
||||
try:
|
||||
sftp.put(local_path, remote_path)
|
||||
finally:
|
||||
sftp.close()
|
||||
|
||||
def read_remote_file(self, remote_path: str) -> str:
|
||||
out, err, code = self.execute(f"cat '{remote_path}'", use_conda=False)
|
||||
if code != 0:
|
||||
raise Exception(f"Failed to read file: {err}")
|
||||
return out
|
||||
|
||||
def write_remote_file(self, remote_path: str, content: str):
|
||||
encoded = base64.b64encode(content.encode()).decode()
|
||||
cmd = f"echo '{encoded}' | base64 -d > '{remote_path}'"
|
||||
out, err, code = self.execute(cmd, use_conda=False)
|
||||
if code != 0:
|
||||
raise Exception(f"Failed to write file: {err}")
|
||||
|
||||
def is_connected(self) -> bool:
|
||||
try:
|
||||
if self.client:
|
||||
transport = self.client.get_transport()
|
||||
if transport and transport.is_active():
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
self.connected = False
|
||||
return False
|
||||
|
||||
|
||||
# Singleton shared across all routes
|
||||
ssh_manager = SSHClient()
|
||||
29
docker-compose.yml
Normal file
29
docker-compose.yml
Normal file
@@ -0,0 +1,29 @@
|
||||
version: "3.9"
|
||||
|
||||
services:
|
||||
|
||||
backend:
|
||||
build: ./backend
|
||||
container_name: llm-trainer-backend
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8080:8080"
|
||||
environment:
|
||||
- OLLAMA_URL=http://192.168.2.47:11434
|
||||
networks:
|
||||
- llm-net
|
||||
|
||||
frontend:
|
||||
build: ./frontend
|
||||
container_name: llm-trainer-frontend
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "3000:80"
|
||||
depends_on:
|
||||
- backend
|
||||
networks:
|
||||
- llm-net
|
||||
|
||||
networks:
|
||||
llm-net:
|
||||
driver: bridge
|
||||
14
frontend/Dockerfile
Normal file
14
frontend/Dockerfile
Normal file
@@ -0,0 +1,14 @@
|
||||
FROM node:20-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
COPY package.json .
|
||||
RUN npm install
|
||||
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
|
||||
FROM nginx:alpine
|
||||
COPY --from=builder /app/dist /usr/share/nginx/html
|
||||
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
||||
EXPOSE 80
|
||||
CMD ["nginx", "-g", "daemon off;"]
|
||||
18
frontend/index.html
Normal file
18
frontend/index.html
Normal file
@@ -0,0 +1,18 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>LLM Trainer Dashboard</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link
|
||||
href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;500;600&display=swap"
|
||||
rel="stylesheet"
|
||||
/>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/index.jsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
22
frontend/nginx.conf
Normal file
22
frontend/nginx.conf
Normal file
@@ -0,0 +1,22 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
# Proxy API and WebSocket calls to the backend
|
||||
location /api/ {
|
||||
proxy_pass http://backend:8080;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_set_header Host $host;
|
||||
proxy_read_timeout 3600;
|
||||
}
|
||||
|
||||
# SPA fallback
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
28
frontend/package.json
Normal file
28
frontend/package.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "llm-trainer-frontend",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"react": "^18.2.0",
|
||||
"react-dom": "^18.2.0",
|
||||
"axios": "^1.6.7",
|
||||
"@xterm/xterm": "^5.3.0",
|
||||
"@xterm/addon-fit": "^0.8.0",
|
||||
"@xterm/addon-web-links": "^0.9.0",
|
||||
"recharts": "^2.10.3",
|
||||
"react-dropzone": "^14.2.3",
|
||||
"lucide-react": "^0.344.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@vitejs/plugin-react": "^4.2.1",
|
||||
"autoprefixer": "^10.4.17",
|
||||
"postcss": "^8.4.35",
|
||||
"tailwindcss": "^3.4.1",
|
||||
"vite": "^5.1.4"
|
||||
}
|
||||
}
|
||||
6
frontend/postcss.config.js
Normal file
6
frontend/postcss.config.js
Normal file
@@ -0,0 +1,6 @@
|
||||
export default {
|
||||
plugins: {
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
}
|
||||
141
frontend/src/App.jsx
Normal file
141
frontend/src/App.jsx
Normal file
@@ -0,0 +1,141 @@
|
||||
import React, { useState, useEffect, useCallback } from 'react'
|
||||
import axios from 'axios'
|
||||
import {
|
||||
Wifi, WifiOff, Server, FileText, GitBranch,
|
||||
Table2, Activity, TerminalSquare, Box, Settings, RefreshCw,
|
||||
} from 'lucide-react'
|
||||
|
||||
import ConnectionPanel from './components/ConnectionPanel'
|
||||
import DocumentManager from './components/DocumentManager'
|
||||
import PipelineRunner from './components/PipelineRunner'
|
||||
import QAPairViewer from './components/QAPairViewer'
|
||||
import TrainingMonitor from './components/TrainingMonitor'
|
||||
import Terminal from './components/Terminal'
|
||||
import ModelManager from './components/ModelManager'
|
||||
import ConfigEditor from './components/ConfigEditor'
|
||||
|
||||
const API = '' // vite proxy forwards /api → :8080
|
||||
|
||||
const NAV = [
|
||||
{ id: 'connection', label: 'Connection', icon: Server },
|
||||
{ id: 'documents', label: 'Documents', icon: FileText },
|
||||
{ id: 'pipeline', label: 'Pipeline', icon: GitBranch },
|
||||
{ id: 'pairs', label: 'QA Pairs', icon: Table2 },
|
||||
{ id: 'training', label: 'Training', icon: Activity },
|
||||
{ id: 'terminal', label: 'Terminal', icon: TerminalSquare },
|
||||
{ id: 'models', label: 'Models', icon: Box },
|
||||
{ id: 'config', label: 'Config', icon: Settings },
|
||||
]
|
||||
|
||||
export default function App() {
|
||||
const [active, setActive] = useState('connection')
|
||||
const [connected, setConnected] = useState(false)
|
||||
const [gpuInfo, setGpuInfo] = useState(null)
|
||||
const [statusMsg, setStatusMsg] = useState('')
|
||||
|
||||
const fetchStatus = useCallback(async () => {
|
||||
try {
|
||||
const { data } = await axios.get(`${API}/api/status`)
|
||||
setConnected(data.connected)
|
||||
if (data.gpu?.gpus?.length) setGpuInfo(data.gpu.gpus[0])
|
||||
} catch {
|
||||
setConnected(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
fetchStatus()
|
||||
const id = setInterval(fetchStatus, 10000)
|
||||
return () => clearInterval(id)
|
||||
}, [fetchStatus])
|
||||
|
||||
const panels = {
|
||||
connection: <ConnectionPanel onConnect={fetchStatus} setStatus={setStatusMsg} />,
|
||||
documents: <DocumentManager connected={connected} />,
|
||||
pipeline: <PipelineRunner connected={connected} />,
|
||||
pairs: <QAPairViewer connected={connected} />,
|
||||
training: <TrainingMonitor connected={connected} gpuInfo={gpuInfo} />,
|
||||
terminal: <Terminal connected={connected} />,
|
||||
models: <ModelManager connected={connected} />,
|
||||
config: <ConfigEditor connected={connected} />,
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex h-screen overflow-hidden bg-[#0f1117] text-slate-200">
|
||||
|
||||
{/* ── Sidebar ── */}
|
||||
<aside className="w-52 flex-shrink-0 flex flex-col bg-[#161b27] border-r border-slate-700/50">
|
||||
{/* Logo */}
|
||||
<div className="px-4 py-5 border-b border-slate-700/50">
|
||||
<h1 className="text-sm font-semibold tracking-widest text-blue-400 uppercase">
|
||||
LLM Trainer
|
||||
</h1>
|
||||
<p className="text-xs text-slate-500 mt-0.5">Dashboard v1.0</p>
|
||||
</div>
|
||||
|
||||
{/* Nav items */}
|
||||
<nav className="flex-1 py-3 overflow-y-auto">
|
||||
{NAV.map(({ id, label, icon: Icon }) => (
|
||||
<button
|
||||
key={id}
|
||||
onClick={() => setActive(id)}
|
||||
className={`w-full flex items-center gap-3 px-4 py-2.5 text-sm transition-colors
|
||||
${active === id
|
||||
? 'bg-blue-600/20 text-blue-400 border-r-2 border-blue-500'
|
||||
: 'text-slate-400 hover:text-slate-200 hover:bg-slate-700/30'}`}
|
||||
>
|
||||
<Icon size={16} />
|
||||
{label}
|
||||
</button>
|
||||
))}
|
||||
</nav>
|
||||
|
||||
{/* Status bar */}
|
||||
<div className="px-4 py-3 border-t border-slate-700/50 space-y-2">
|
||||
<div className={`flex items-center gap-2 text-xs font-medium
|
||||
${connected ? 'text-green-400' : 'text-red-400'}`}>
|
||||
{connected
|
||||
? <><Wifi size={12}/> Connected</>
|
||||
: <><WifiOff size={12}/> Disconnected</>
|
||||
}
|
||||
</div>
|
||||
{gpuInfo && connected && (
|
||||
<div className="text-xs text-slate-500 space-y-0.5">
|
||||
<div className="truncate">{gpuInfo.name}</div>
|
||||
<div className="flex gap-2">
|
||||
<span className="text-purple-400">{gpuInfo.utilization}%</span>
|
||||
<span className="text-blue-400">
|
||||
{gpuInfo.memory_used}/{gpuInfo.memory_total}MB
|
||||
</span>
|
||||
<span className="text-orange-400">{gpuInfo.temperature}°C</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<button
|
||||
onClick={fetchStatus}
|
||||
className="flex items-center gap-1.5 text-xs text-slate-500 hover:text-slate-300 transition-colors"
|
||||
>
|
||||
<RefreshCw size={11}/> Refresh
|
||||
</button>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
{/* ── Main content ── */}
|
||||
<main className="flex-1 flex flex-col overflow-hidden">
|
||||
{/* Top bar */}
|
||||
<header className="flex items-center justify-between px-6 py-3 border-b border-slate-700/50 bg-[#161b27]">
|
||||
<h2 className="text-sm font-semibold text-slate-300 capitalize">
|
||||
{NAV.find(n => n.id === active)?.label}
|
||||
</h2>
|
||||
{statusMsg && (
|
||||
<span className="text-xs text-slate-500 italic">{statusMsg}</span>
|
||||
)}
|
||||
</header>
|
||||
|
||||
<div className="flex-1 overflow-auto p-6">
|
||||
{panels[active]}
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
192
frontend/src/components/ConfigEditor.jsx
Normal file
192
frontend/src/components/ConfigEditor.jsx
Normal file
@@ -0,0 +1,192 @@
|
||||
import React, { useState, useEffect } from 'react'
|
||||
import axios from 'axios'
|
||||
import { Save, RefreshCw, ChevronDown, ChevronRight } from 'lucide-react'
|
||||
|
||||
function Field({ label, value, onChange, type = 'text', options = null }) {
|
||||
return (
|
||||
<div>
|
||||
<label className="block text-xs text-slate-500 mb-1">{label}</label>
|
||||
{options ? (
|
||||
<select
|
||||
value={value ?? ''}
|
||||
onChange={e => onChange(e.target.value)}
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-1.5 text-xs
|
||||
text-slate-200 focus:outline-none focus:border-blue-500"
|
||||
>
|
||||
{options.map(o => <option key={o} value={o}>{o}</option>)}
|
||||
</select>
|
||||
) : (
|
||||
<input
|
||||
type={type}
|
||||
value={value ?? ''}
|
||||
onChange={e => onChange(type === 'number' ? Number(e.target.value) : e.target.value)}
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-1.5 text-xs
|
||||
text-slate-200 focus:outline-none focus:border-blue-500"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Section({ title, children, defaultOpen = true }) {
|
||||
const [open, setOpen] = useState(defaultOpen)
|
||||
return (
|
||||
<div className="rounded-xl border border-slate-700/50 bg-[#161b27] overflow-hidden">
|
||||
<button
|
||||
onClick={() => setOpen(o => !o)}
|
||||
className="w-full flex items-center justify-between px-4 py-3 text-xs font-semibold
|
||||
text-slate-300 hover:bg-slate-700/20 transition-colors"
|
||||
>
|
||||
{title}
|
||||
{open ? <ChevronDown size={13}/> : <ChevronRight size={13}/>}
|
||||
</button>
|
||||
{open && <div className="px-4 pb-4 grid grid-cols-2 gap-3">{children}</div>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default function ConfigEditor({ connected }) {
|
||||
const [config, setConfig] = useState(null)
|
||||
const [raw, setRaw] = useState('')
|
||||
const [mode, setMode] = useState('form') // 'form' | 'raw'
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [saving, setSaving] = useState(false)
|
||||
const [msg, setMsg] = useState('')
|
||||
|
||||
const load = async () => {
|
||||
if (!connected) return
|
||||
setLoading(true); setMsg('')
|
||||
try {
|
||||
const { data } = await axios.get('/api/config')
|
||||
setConfig(data.config || {})
|
||||
setRaw(data.raw || '')
|
||||
} catch (err) {
|
||||
setMsg(`Load failed: ${err.response?.data?.detail || err.message}`)
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => { load() }, [connected])
|
||||
|
||||
const set = (path, val) => {
|
||||
setConfig(c => {
|
||||
const updated = { ...c }
|
||||
const keys = path.split('.')
|
||||
let cur = updated
|
||||
for (let i = 0; i < keys.length - 1; i++) {
|
||||
cur[keys[i]] = { ...(cur[keys[i]] || {}) }
|
||||
cur = cur[keys[i]]
|
||||
}
|
||||
cur[keys[keys.length - 1]] = val
|
||||
return updated
|
||||
})
|
||||
}
|
||||
|
||||
const g = path => {
|
||||
if (!config) return ''
|
||||
return path.split('.').reduce((o, k) => (o || {})[k], config) ?? ''
|
||||
}
|
||||
|
||||
const save = async () => {
|
||||
setSaving(true); setMsg('')
|
||||
try {
|
||||
const payload = mode === 'raw'
|
||||
? await axios.put('/api/config', config) // fallback
|
||||
: await axios.put('/api/config', config)
|
||||
setMsg('Configuration saved ✓')
|
||||
} catch (err) {
|
||||
setMsg(`Save failed: ${err.response?.data?.detail || err.message}`)
|
||||
} finally {
|
||||
setSaving(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
|
||||
{/* Toolbar */}
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex gap-1 bg-[#161b27] rounded-xl p-1">
|
||||
{['form', 'raw'].map(m => (
|
||||
<button key={m} onClick={() => setMode(m)}
|
||||
className={`px-4 py-1.5 rounded-lg text-xs font-medium transition-colors capitalize
|
||||
${mode === m ? 'bg-blue-600 text-white' : 'text-slate-400 hover:text-slate-200'}`}>
|
||||
{m === 'form' ? 'Form Editor' : 'Raw YAML'}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<button onClick={load} disabled={!connected}
|
||||
className="flex items-center gap-1 text-xs text-slate-500 hover:text-slate-300 disabled:opacity-40 transition-colors">
|
||||
<RefreshCw size={12}/> Reload
|
||||
</button>
|
||||
<button onClick={save} disabled={saving || !connected}
|
||||
className="flex items-center gap-2 px-4 py-2 rounded-lg text-xs font-medium
|
||||
bg-blue-600 hover:bg-blue-500 disabled:opacity-40 disabled:cursor-not-allowed transition-colors">
|
||||
<Save size={13}/>
|
||||
{saving ? 'Saving…' : 'Save Config'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{msg && (
|
||||
<p className={`text-xs px-3 py-2 rounded-lg border ${
|
||||
msg.includes('✓')
|
||||
? 'text-green-400 bg-green-900/10 border-green-700/40'
|
||||
: 'text-red-400 bg-red-900/10 border-red-700/40'
|
||||
}`}>{msg}</p>
|
||||
)}
|
||||
|
||||
{!connected ? (
|
||||
<div className="rounded-xl border border-yellow-700/40 bg-yellow-900/10 px-4 py-3 text-xs text-yellow-400">
|
||||
Connect to SSH server to edit config.
|
||||
</div>
|
||||
) : loading ? (
|
||||
<p className="text-xs text-slate-500 text-center py-10">Loading config…</p>
|
||||
) : mode === 'raw' ? (
|
||||
<div>
|
||||
<p className="text-xs text-slate-500 mb-2">
|
||||
Editing <code className="text-blue-400">/opt/synthetic/synthetic-data-kit/config.yaml</code> directly
|
||||
</p>
|
||||
<textarea
|
||||
rows={30}
|
||||
value={raw}
|
||||
onChange={e => setRaw(e.target.value)}
|
||||
className="w-full bg-[#0a0d14] border border-slate-700 rounded-xl px-4 py-3 text-xs
|
||||
font-mono text-slate-200 focus:outline-none focus:border-blue-500 resize-none"
|
||||
/>
|
||||
</div>
|
||||
) : config ? (
|
||||
<div className="space-y-3">
|
||||
|
||||
<Section title="LLM Provider">
|
||||
<Field label="Provider" value={g('llm.provider')} onChange={v => set('llm.provider', v)}
|
||||
options={['openai', 'ollama', 'vllm']}/>
|
||||
<Field label="Model" value={g('llm.model')} onChange={v => set('llm.model', v)}/>
|
||||
<Field label="API Base URL" value={g('llm.api_base')} onChange={v => set('llm.api_base', v)}/>
|
||||
<Field label="Max Tokens" value={g('llm.max_tokens')} onChange={v => set('llm.max_tokens', v)} type="number"/>
|
||||
<Field label="Temperature" value={g('llm.temperature')} onChange={v => set('llm.temperature', v)} type="number"/>
|
||||
</Section>
|
||||
|
||||
<Section title="Generation Settings">
|
||||
<Field label="Default Pairs" value={g('generation.num_pairs')} onChange={v => set('generation.num_pairs', v)} type="number"/>
|
||||
<Field label="Default Type" value={g('generation.pair_type')} onChange={v => set('generation.pair_type', v)}
|
||||
options={['qa', 'summary', 'cot']}/>
|
||||
<Field label="Quality Threshold" value={g('generation.threshold')} onChange={v => set('generation.threshold', v)} type="number"/>
|
||||
<Field label="Batch Size" value={g('generation.batch_size')} onChange={v => set('generation.batch_size', v)} type="number"/>
|
||||
</Section>
|
||||
|
||||
<Section title="Paths" defaultOpen={false}>
|
||||
<Field label="Input Dir" value={g('paths.input')} onChange={v => set('paths.input', v)}/>
|
||||
<Field label="Parsed Dir" value={g('paths.parsed')} onChange={v => set('paths.parsed', v)}/>
|
||||
<Field label="Generated Dir" value={g('paths.generated')} onChange={v => set('paths.generated', v)}/>
|
||||
<Field label="Curated Dir" value={g('paths.curated')} onChange={v => set('paths.curated', v)}/>
|
||||
<Field label="Final Dir" value={g('paths.final')} onChange={v => set('paths.final', v)}/>
|
||||
</Section>
|
||||
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
181
frontend/src/components/ConnectionPanel.jsx
Normal file
181
frontend/src/components/ConnectionPanel.jsx
Normal file
@@ -0,0 +1,181 @@
|
||||
import React, { useState } from 'react'
|
||||
import axios from 'axios'
|
||||
import { Server, LogIn, LogOut, Key, Eye, EyeOff } from 'lucide-react'
|
||||
|
||||
export default function ConnectionPanel({ onConnect, setStatus }) {
|
||||
const [form, setForm] = useState({
|
||||
host: '192.168.2.47',
|
||||
username: 'tocmo0nlord',
|
||||
password: '',
|
||||
key_path: '',
|
||||
port: 22,
|
||||
})
|
||||
const [showPass, setShowPass] = useState(false)
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [connected, setConnected] = useState(false)
|
||||
const [error, setError] = useState('')
|
||||
const [info, setInfo] = useState(null)
|
||||
|
||||
const handleChange = e =>
|
||||
setForm(f => ({ ...f, [e.target.name]: e.target.value }))
|
||||
|
||||
const connect = async () => {
|
||||
setLoading(true); setError('')
|
||||
try {
|
||||
const { data } = await axios.post('/api/connect', {
|
||||
...form,
|
||||
port: Number(form.port),
|
||||
password: form.password || undefined,
|
||||
key_path: form.key_path || undefined,
|
||||
})
|
||||
setConnected(true)
|
||||
setInfo(data)
|
||||
setStatus(`Connected to ${data.host}`)
|
||||
onConnect()
|
||||
} catch (err) {
|
||||
setError(err.response?.data?.detail || err.message)
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
const disconnect = async () => {
|
||||
await axios.post('/api/disconnect').catch(() => {})
|
||||
setConnected(false)
|
||||
setInfo(null)
|
||||
setStatus('Disconnected')
|
||||
onConnect()
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="max-w-xl mx-auto space-y-6">
|
||||
|
||||
{/* Status card */}
|
||||
<div className={`rounded-xl border p-4 flex items-center gap-3
|
||||
${connected
|
||||
? 'border-green-600/40 bg-green-900/10'
|
||||
: 'border-slate-600/40 bg-slate-800/30'}`}>
|
||||
<Server size={20} className={connected ? 'text-green-400' : 'text-slate-500'} />
|
||||
<div>
|
||||
<p className={`text-sm font-semibold ${connected ? 'text-green-400' : 'text-slate-400'}`}>
|
||||
{connected ? 'SSH Connected' : 'Not Connected'}
|
||||
</p>
|
||||
{info && (
|
||||
<p className="text-xs text-slate-400">
|
||||
{info.username}@{info.host}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Form */}
|
||||
<div className="rounded-xl border border-slate-700/50 bg-[#161b27] p-6 space-y-4">
|
||||
<h3 className="text-sm font-semibold text-slate-300 mb-1">SSH Credentials</h3>
|
||||
|
||||
<div className="grid grid-cols-3 gap-3">
|
||||
<div className="col-span-2">
|
||||
<label className="block text-xs text-slate-500 mb-1">Host / IP</label>
|
||||
<input
|
||||
name="host"
|
||||
value={form.host}
|
||||
onChange={handleChange}
|
||||
disabled={connected}
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-2 text-sm
|
||||
text-slate-200 focus:outline-none focus:border-blue-500 disabled:opacity-50"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label className="block text-xs text-slate-500 mb-1">Port</label>
|
||||
<input
|
||||
name="port"
|
||||
type="number"
|
||||
value={form.port}
|
||||
onChange={handleChange}
|
||||
disabled={connected}
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-2 text-sm
|
||||
text-slate-200 focus:outline-none focus:border-blue-500 disabled:opacity-50"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="block text-xs text-slate-500 mb-1">Username</label>
|
||||
<input
|
||||
name="username"
|
||||
value={form.username}
|
||||
onChange={handleChange}
|
||||
disabled={connected}
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-2 text-sm
|
||||
text-slate-200 focus:outline-none focus:border-blue-500 disabled:opacity-50"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="block text-xs text-slate-500 mb-1">Password</label>
|
||||
<div className="relative">
|
||||
<input
|
||||
name="password"
|
||||
type={showPass ? 'text' : 'password'}
|
||||
value={form.password}
|
||||
onChange={handleChange}
|
||||
disabled={connected}
|
||||
placeholder="Leave blank to use SSH key"
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-2 pr-10 text-sm
|
||||
text-slate-200 focus:outline-none focus:border-blue-500 disabled:opacity-50"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setShowPass(s => !s)}
|
||||
className="absolute right-3 top-2.5 text-slate-500 hover:text-slate-300"
|
||||
>
|
||||
{showPass ? <EyeOff size={15}/> : <Eye size={15}/>}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="block text-xs text-slate-500 mb-1">
|
||||
<Key size={11} className="inline mr-1"/>SSH Key Path (optional)
|
||||
</label>
|
||||
<input
|
||||
name="key_path"
|
||||
value={form.key_path}
|
||||
onChange={handleChange}
|
||||
disabled={connected}
|
||||
placeholder="/home/user/.ssh/id_rsa"
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-2 text-sm
|
||||
text-slate-200 focus:outline-none focus:border-blue-500 disabled:opacity-50"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<p className="text-xs text-red-400 bg-red-900/20 border border-red-700/40 rounded-lg px-3 py-2">
|
||||
{error}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{connected ? (
|
||||
<button
|
||||
onClick={disconnect}
|
||||
className="w-full flex items-center justify-center gap-2 py-2.5 rounded-lg
|
||||
bg-red-600/20 border border-red-600/40 text-red-400 hover:bg-red-600/30
|
||||
text-sm font-medium transition-colors"
|
||||
>
|
||||
<LogOut size={15}/> Disconnect
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
onClick={connect}
|
||||
disabled={loading}
|
||||
className="w-full flex items-center justify-center gap-2 py-2.5 rounded-lg
|
||||
bg-blue-600 hover:bg-blue-500 disabled:opacity-50 disabled:cursor-not-allowed
|
||||
text-sm font-medium transition-colors"
|
||||
>
|
||||
<LogIn size={15}/>
|
||||
{loading ? 'Connecting…' : 'Connect'}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
187
frontend/src/components/DocumentManager.jsx
Normal file
187
frontend/src/components/DocumentManager.jsx
Normal file
@@ -0,0 +1,187 @@
|
||||
import React, { useState, useEffect, useCallback } from 'react'
|
||||
import axios from 'axios'
|
||||
import { useDropzone } from 'react-dropzone'
|
||||
import { Upload, Trash2, Eye, RefreshCw, FileText, X } from 'lucide-react'
|
||||
|
||||
const STAGES = ['input', 'parsed', 'generated', 'curated', 'final']
|
||||
|
||||
function formatBytes(bytes) {
|
||||
if (bytes < 1024) return `${bytes} B`
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`
|
||||
}
|
||||
|
||||
export default function DocumentManager({ connected }) {
|
||||
const [stage, setStage] = useState('input')
|
||||
const [files, setFiles] = useState([])
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [uploading, setUploading] = useState(false)
|
||||
const [preview, setPreview] = useState(null)
|
||||
const [error, setError] = useState('')
|
||||
|
||||
const fetchFiles = useCallback(async () => {
|
||||
if (!connected) return
|
||||
setLoading(true); setError('')
|
||||
try {
|
||||
const { data } = await axios.get(`/api/files/${stage}`)
|
||||
setFiles(data.files)
|
||||
} catch (err) {
|
||||
setError(err.response?.data?.detail || err.message)
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}, [stage, connected])
|
||||
|
||||
useEffect(() => { fetchFiles() }, [fetchFiles])
|
||||
|
||||
const onDrop = useCallback(async accepted => {
|
||||
if (!accepted.length || !connected) return
|
||||
setUploading(true)
|
||||
const formData = new FormData()
|
||||
formData.append('file', accepted[0])
|
||||
try {
|
||||
await axios.post('/api/upload', formData)
|
||||
fetchFiles()
|
||||
} catch (err) {
|
||||
setError(err.response?.data?.detail || err.message)
|
||||
} finally {
|
||||
setUploading(false)
|
||||
}
|
||||
}, [connected, fetchFiles])
|
||||
|
||||
const { getRootProps, getInputProps, isDragActive } = useDropzone({
|
||||
onDrop,
|
||||
disabled: !connected || stage !== 'input',
|
||||
multiple: false,
|
||||
})
|
||||
|
||||
const deleteFile = async name => {
|
||||
if (!window.confirm(`Delete "${name}"?`)) return
|
||||
await axios.delete(`/api/files/${stage}/${name}`)
|
||||
fetchFiles()
|
||||
}
|
||||
|
||||
const previewFile = async name => {
|
||||
const { data } = await axios.get(`/api/files/${stage}/${name}/preview`)
|
||||
setPreview({ name, content: data.content })
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
|
||||
{/* Stage tabs */}
|
||||
<div className="flex gap-1 bg-[#161b27] rounded-xl p-1 w-fit">
|
||||
{STAGES.map(s => (
|
||||
<button
|
||||
key={s}
|
||||
onClick={() => setStage(s)}
|
||||
className={`px-4 py-1.5 rounded-lg text-xs font-medium transition-colors capitalize
|
||||
${stage === s
|
||||
? 'bg-blue-600 text-white'
|
||||
: 'text-slate-400 hover:text-slate-200'}`}
|
||||
>
|
||||
{s}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Upload drop zone (input stage only) */}
|
||||
{stage === 'input' && (
|
||||
<div
|
||||
{...getRootProps()}
|
||||
className={`border-2 border-dashed rounded-xl p-8 text-center transition-colors cursor-pointer
|
||||
${isDragActive ? 'border-blue-500 bg-blue-900/10' : 'border-slate-600/50 hover:border-slate-500'}
|
||||
${!connected ? 'opacity-40 cursor-not-allowed' : ''}`}
|
||||
>
|
||||
<input {...getInputProps()} />
|
||||
<Upload size={24} className="mx-auto mb-2 text-slate-500" />
|
||||
{uploading
|
||||
? <p className="text-sm text-blue-400">Uploading…</p>
|
||||
: <p className="text-sm text-slate-400">
|
||||
{isDragActive ? 'Drop file here' : 'Drag & drop or click to upload to /input'}
|
||||
</p>
|
||||
}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* File table */}
|
||||
<div className="rounded-xl border border-slate-700/50 bg-[#161b27] overflow-hidden">
|
||||
<div className="flex items-center justify-between px-4 py-3 border-b border-slate-700/50">
|
||||
<span className="text-xs text-slate-400">{files.length} file(s)</span>
|
||||
<button
|
||||
onClick={fetchFiles}
|
||||
className="flex items-center gap-1 text-xs text-slate-500 hover:text-slate-300 transition-colors"
|
||||
>
|
||||
<RefreshCw size={12}/> Refresh
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<p className="px-4 py-2 text-xs text-red-400">{error}</p>
|
||||
)}
|
||||
|
||||
{loading ? (
|
||||
<p className="px-4 py-6 text-xs text-slate-500 text-center">Loading…</p>
|
||||
) : files.length === 0 ? (
|
||||
<p className="px-4 py-6 text-xs text-slate-500 text-center">No files in /{stage}</p>
|
||||
) : (
|
||||
<table className="w-full text-xs">
|
||||
<thead>
|
||||
<tr className="text-slate-500 border-b border-slate-700/40">
|
||||
<th className="text-left px-4 py-2">Name</th>
|
||||
<th className="text-right px-4 py-2">Size</th>
|
||||
<th className="text-right px-4 py-2">Modified</th>
|
||||
<th className="px-4 py-2"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{files.map(f => (
|
||||
<tr key={f.name} className="border-b border-slate-700/20 hover:bg-slate-700/10">
|
||||
<td className="px-4 py-2 flex items-center gap-2">
|
||||
<FileText size={13} className="text-slate-500 flex-shrink-0"/>
|
||||
<span className="truncate max-w-xs text-slate-200">{f.name}</span>
|
||||
</td>
|
||||
<td className="px-4 py-2 text-right text-slate-400">{formatBytes(f.size)}</td>
|
||||
<td className="px-4 py-2 text-right text-slate-500">{f.modified}</td>
|
||||
<td className="px-4 py-2 text-right">
|
||||
<div className="flex items-center justify-end gap-2">
|
||||
<button
|
||||
onClick={() => previewFile(f.name)}
|
||||
className="p-1 rounded hover:bg-blue-600/20 text-slate-500 hover:text-blue-400 transition-colors"
|
||||
>
|
||||
<Eye size={13}/>
|
||||
</button>
|
||||
<button
|
||||
onClick={() => deleteFile(f.name)}
|
||||
className="p-1 rounded hover:bg-red-600/20 text-slate-500 hover:text-red-400 transition-colors"
|
||||
>
|
||||
<Trash2 size={13}/>
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Preview modal */}
|
||||
{preview && (
|
||||
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/60 p-6">
|
||||
<div className="bg-[#161b27] border border-slate-700 rounded-xl w-full max-w-3xl max-h-[80vh] flex flex-col">
|
||||
<div className="flex items-center justify-between px-4 py-3 border-b border-slate-700">
|
||||
<span className="text-sm font-medium text-slate-200">{preview.name}</span>
|
||||
<button onClick={() => setPreview(null)} className="text-slate-500 hover:text-slate-200">
|
||||
<X size={16}/>
|
||||
</button>
|
||||
</div>
|
||||
<pre className="flex-1 overflow-auto p-4 text-xs text-slate-300 leading-relaxed whitespace-pre-wrap">
|
||||
{preview.content}
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
170
frontend/src/components/ModelManager.jsx
Normal file
170
frontend/src/components/ModelManager.jsx
Normal file
@@ -0,0 +1,170 @@
|
||||
import React, { useState, useEffect, useRef } from 'react'
|
||||
import axios from 'axios'
|
||||
import { Box, Download, Trash2, RefreshCw, CheckCircle2, Loader2, XCircle } from 'lucide-react'
|
||||
|
||||
const WS_BASE = `ws://${window.location.host}`
|
||||
|
||||
function formatSize(bytes) {
|
||||
if (!bytes) return '—'
|
||||
const gb = bytes / (1024 ** 3)
|
||||
return gb >= 1 ? `${gb.toFixed(1)} GB` : `${(bytes / (1024 ** 2)).toFixed(0)} MB`
|
||||
}
|
||||
|
||||
function PullProgress({ progress }) {
|
||||
const pct = progress.total > 0 ? Math.round((progress.completed / progress.total) * 100) : 0
|
||||
return (
|
||||
<div className="space-y-1">
|
||||
<div className="flex justify-between text-xs text-slate-400">
|
||||
<span className="truncate">{progress.status || 'Pulling…'}</span>
|
||||
<span>{pct}%</span>
|
||||
</div>
|
||||
<div className="h-1.5 bg-slate-700 rounded-full overflow-hidden">
|
||||
<div className="h-full bg-blue-500 rounded-full transition-all" style={{ width: `${pct}%` }}/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default function ModelManager({ connected }) {
|
||||
const [models, setModels] = useState([])
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [pullName, setPullName] = useState('')
|
||||
const [pulling, setPulling] = useState(false)
|
||||
const [pullStatus, setPullStatus] = useState(null) // null | { status, completed, total }
|
||||
const [pullResult, setPullResult] = useState(null) // 'success' | 'error'
|
||||
const [pullError, setPullError] = useState('')
|
||||
const wsRef = useRef(null)
|
||||
|
||||
const fetchModels = async () => {
|
||||
if (!connected) return
|
||||
setLoading(true)
|
||||
try {
|
||||
const { data } = await axios.get('/api/models')
|
||||
setModels(data.models)
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => { fetchModels() }, [connected])
|
||||
|
||||
const pullModel = () => {
|
||||
if (!pullName.trim()) return
|
||||
if (wsRef.current) wsRef.current.close()
|
||||
setPulling(true); setPullResult(null); setPullError(''); setPullStatus(null)
|
||||
|
||||
const ws = new WebSocket(`${WS_BASE}/api/models/pull?model_name=${encodeURIComponent(pullName)}`)
|
||||
wsRef.current = ws
|
||||
|
||||
ws.onmessage = e => {
|
||||
try {
|
||||
const msg = JSON.parse(e.data)
|
||||
if (msg.error) { setPullError(msg.error); setPullResult('error'); setPulling(false); return }
|
||||
if (msg.status === 'success') { setPullResult('success'); setPulling(false); fetchModels(); return }
|
||||
setPullStatus(msg)
|
||||
} catch {}
|
||||
}
|
||||
ws.onerror = () => { setPullResult('error'); setPulling(false) }
|
||||
}
|
||||
|
||||
const deleteModel = async name => {
|
||||
if (!window.confirm(`Delete model "${name}"?`)) return
|
||||
try {
|
||||
await axios.delete(`/api/models/${encodeURIComponent(name)}`)
|
||||
fetchModels()
|
||||
} catch (err) {
|
||||
alert(err.response?.data?.detail || err.message)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-5">
|
||||
|
||||
{/* Pull model */}
|
||||
<div className="rounded-xl border border-slate-700/50 bg-[#161b27] p-5 space-y-3">
|
||||
<h3 className="text-xs font-semibold text-slate-400 uppercase tracking-wider">Pull Model</h3>
|
||||
<div className="flex gap-2">
|
||||
<input
|
||||
value={pullName}
|
||||
onChange={e => setPullName(e.target.value)}
|
||||
onKeyDown={e => e.key === 'Enter' && pullModel()}
|
||||
placeholder="llama3.1:8b, mistral:7b, qwen2:7b…"
|
||||
className="flex-1 bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-2 text-xs
|
||||
text-slate-200 focus:outline-none focus:border-blue-500"
|
||||
/>
|
||||
<button
|
||||
onClick={pullModel}
|
||||
disabled={pulling || !pullName.trim()}
|
||||
className="flex items-center gap-2 px-4 py-2 rounded-lg text-xs font-medium
|
||||
bg-blue-600 hover:bg-blue-500 disabled:opacity-40 disabled:cursor-not-allowed transition-colors"
|
||||
>
|
||||
{pulling
|
||||
? <><Loader2 size={13} className="animate-spin"/> Pulling…</>
|
||||
: <><Download size={13}/> Pull</>
|
||||
}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{pullStatus && pulling && <PullProgress progress={pullStatus}/>}
|
||||
|
||||
{pullResult === 'success' && (
|
||||
<div className="flex items-center gap-2 text-xs text-green-400">
|
||||
<CheckCircle2 size={13}/> Model pulled successfully
|
||||
</div>
|
||||
)}
|
||||
{pullResult === 'error' && (
|
||||
<div className="flex items-center gap-2 text-xs text-red-400">
|
||||
<XCircle size={13}/> {pullError || 'Pull failed'}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Model list */}
|
||||
<div className="rounded-xl border border-slate-700/50 bg-[#161b27] overflow-hidden">
|
||||
<div className="flex items-center justify-between px-4 py-3 border-b border-slate-700/50">
|
||||
<span className="text-xs text-slate-400">{models.length} model(s) on Ollama</span>
|
||||
<button
|
||||
onClick={fetchModels}
|
||||
className="flex items-center gap-1 text-xs text-slate-500 hover:text-slate-300 transition-colors"
|
||||
>
|
||||
<RefreshCw size={12}/> Refresh
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{loading ? (
|
||||
<p className="py-8 text-center text-xs text-slate-500">Loading…</p>
|
||||
) : models.length === 0 ? (
|
||||
<p className="py-8 text-center text-xs text-slate-600">
|
||||
{connected ? 'No models found on Ollama' : 'Connect to SSH server first'}
|
||||
</p>
|
||||
) : (
|
||||
<div className="divide-y divide-slate-700/30">
|
||||
{models.map(m => (
|
||||
<div key={m.name} className="flex items-center justify-between px-4 py-3 hover:bg-slate-700/10">
|
||||
<div className="flex items-center gap-3">
|
||||
<Box size={15} className="text-blue-400 flex-shrink-0"/>
|
||||
<div>
|
||||
<p className="text-xs font-medium text-slate-200">{m.name}</p>
|
||||
<p className="text-xs text-slate-500">
|
||||
{formatSize(m.size)}
|
||||
{m.details?.parameter_size && ` · ${m.details.parameter_size}`}
|
||||
{m.details?.quantization_level && ` · ${m.details.quantization_level}`}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => deleteModel(m.name)}
|
||||
className="p-1.5 rounded hover:bg-red-600/20 text-slate-500 hover:text-red-400 transition-colors"
|
||||
>
|
||||
<Trash2 size={14}/>
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
218
frontend/src/components/PipelineRunner.jsx
Normal file
218
frontend/src/components/PipelineRunner.jsx
Normal file
@@ -0,0 +1,218 @@
|
||||
import React, { useState, useRef, useEffect } from 'react'
|
||||
import axios from 'axios'
|
||||
import { Play, CheckCircle2, XCircle, Loader2, ChevronDown } from 'lucide-react'
|
||||
|
||||
const WS_BASE = `ws://${window.location.host}`
|
||||
|
||||
const STAGES = [
|
||||
{
|
||||
id: 'ingest',
|
||||
label: '1 · Ingest',
|
||||
description: 'Parse raw document → text chunks',
|
||||
sourceStage: 'input',
|
||||
endpoint: '/api/pipeline/ingest',
|
||||
params: [{ name: 'filename', label: 'Input file', type: 'select', stage: 'input' }],
|
||||
},
|
||||
{
|
||||
id: 'create',
|
||||
label: '2 · Create',
|
||||
description: 'Generate QA / summary / CoT pairs',
|
||||
sourceStage: 'parsed',
|
||||
endpoint: '/api/pipeline/create',
|
||||
params: [
|
||||
{ name: 'filename', label: 'Parsed file', type: 'select', stage: 'parsed' },
|
||||
{ name: 'num_pairs', label: 'Pair count', type: 'number', default: 50 },
|
||||
{ name: 'pair_type', label: 'Pair type', type: 'select_static',
|
||||
options: ['qa', 'summary', 'cot'] },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'curate',
|
||||
label: '3 · Curate',
|
||||
description: 'Filter pairs by quality score',
|
||||
sourceStage: 'generated',
|
||||
endpoint: '/api/pipeline/curate',
|
||||
params: [
|
||||
{ name: 'filename', label: 'Generated file', type: 'select', stage: 'generated' },
|
||||
{ name: 'output_filename', label: 'Output name', type: 'text', default: 'curated.jsonl' },
|
||||
{ name: 'threshold', label: 'Min score (0–10)', type: 'number', default: 7 },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'save',
|
||||
label: '4 · Save As',
|
||||
description: 'Export final dataset',
|
||||
sourceStage: 'curated',
|
||||
endpoint: '/api/pipeline/save',
|
||||
params: [
|
||||
{ name: 'filename', label: 'Curated file', type: 'select', stage: 'curated' },
|
||||
{ name: 'output_filename', label: 'Output name', type: 'text', default: 'dataset.jsonl' },
|
||||
{ name: 'fmt', label: 'Format', type: 'select_static',
|
||||
options: ['jsonl', 'csv', 'alpaca'] },
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
function LogViewer({ lines }) {
|
||||
const ref = useRef(null)
|
||||
useEffect(() => { ref.current?.scrollTo(0, ref.current.scrollHeight) }, [lines])
|
||||
|
||||
return (
|
||||
<div ref={ref} className="h-48 overflow-y-auto bg-[#0a0d14] rounded-lg p-3 font-mono text-xs leading-relaxed">
|
||||
{lines.length === 0
|
||||
? <span className="text-slate-600">No output yet…</span>
|
||||
: lines.map((l, i) => {
|
||||
const cls = l.includes('Error') || l.includes('error') || l.includes('Traceback')
|
||||
? 'text-red-400'
|
||||
: l.includes('Warning') || l.includes('warning')
|
||||
? 'text-yellow-400'
|
||||
: l.includes('✓') || l.includes('done') || l.includes('Done')
|
||||
? 'text-green-400'
|
||||
: 'text-slate-300'
|
||||
return <div key={i} className={cls}>{l}</div>
|
||||
})
|
||||
}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function StageCard({ stage, connected }) {
|
||||
const [files, setFiles] = useState([])
|
||||
const [values, setValues] = useState(() =>
|
||||
Object.fromEntries(
|
||||
stage.params.map(p => [p.name, p.default ?? ''])
|
||||
)
|
||||
)
|
||||
const [logs, setLogs] = useState([])
|
||||
const [status, setStatus] = useState('idle') // idle | running | done | error
|
||||
const wsRef = useRef(null)
|
||||
|
||||
useEffect(() => {
|
||||
if (!connected) return
|
||||
const loadStage = async p => {
|
||||
try {
|
||||
const { data } = await axios.get(`/api/files/${p.stage}`)
|
||||
return data.files.map(f => f.name)
|
||||
} catch { return [] }
|
||||
}
|
||||
|
||||
Promise.all(
|
||||
stage.params
|
||||
.filter(p => p.type === 'select')
|
||||
.map(p => loadStage(p).then(names => ({ name: p.name, names })))
|
||||
).then(results => {
|
||||
results.forEach(({ name, names }) => {
|
||||
setFiles(prev => ({ ...prev, [name]: names }))
|
||||
setValues(prev => ({ ...prev, [name]: names[0] || '' }))
|
||||
})
|
||||
})
|
||||
}, [connected, stage])
|
||||
|
||||
const run = () => {
|
||||
if (wsRef.current) wsRef.current.close()
|
||||
setLogs([])
|
||||
setStatus('running')
|
||||
|
||||
const params = new URLSearchParams(values).toString()
|
||||
const url = `${WS_BASE}${stage.endpoint}?${params}`
|
||||
const ws = new WebSocket(url)
|
||||
wsRef.current = ws
|
||||
|
||||
ws.onmessage = e => {
|
||||
const msg = JSON.parse(e.data)
|
||||
if (msg.type === 'log') setLogs(l => [...l, msg.data])
|
||||
if (msg.type === 'done') { setStatus('done'); ws.close() }
|
||||
if (msg.type === 'error') { setStatus('error'); setLogs(l => [...l, `ERROR: ${msg.data}`]); ws.close() }
|
||||
}
|
||||
ws.onerror = () => setStatus('error')
|
||||
ws.onclose = () => { if (status === 'running') setStatus('done') }
|
||||
}
|
||||
|
||||
const setValue = (name, val) => setValues(v => ({ ...v, [name]: val }))
|
||||
|
||||
const statusIcon = {
|
||||
idle: null,
|
||||
running: <Loader2 size={14} className="text-blue-400 animate-spin"/>,
|
||||
done: <CheckCircle2 size={14} className="text-green-400"/>,
|
||||
error: <XCircle size={14} className="text-red-400"/>,
|
||||
}
|
||||
|
||||
return (
|
||||
<div className={`rounded-xl border bg-[#161b27] p-5 space-y-4
|
||||
${status === 'done' ? 'border-green-700/40' : status === 'error' ? 'border-red-700/40' : 'border-slate-700/50'}`}>
|
||||
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h3 className="text-sm font-semibold text-slate-200">{stage.label}</h3>
|
||||
<p className="text-xs text-slate-500 mt-0.5">{stage.description}</p>
|
||||
</div>
|
||||
{statusIcon[status]}
|
||||
</div>
|
||||
|
||||
{/* Params */}
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
{stage.params.map(p => (
|
||||
<div key={p.name}>
|
||||
<label className="block text-xs text-slate-500 mb-1">{p.label}</label>
|
||||
{p.type === 'select' ? (
|
||||
<select
|
||||
value={values[p.name]}
|
||||
onChange={e => setValue(p.name, e.target.value)}
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-1.5 text-xs
|
||||
text-slate-200 focus:outline-none focus:border-blue-500"
|
||||
>
|
||||
{(files[p.name] || []).map(f => <option key={f} value={f}>{f}</option>)}
|
||||
</select>
|
||||
) : p.type === 'select_static' ? (
|
||||
<select
|
||||
value={values[p.name]}
|
||||
onChange={e => setValue(p.name, e.target.value)}
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-1.5 text-xs
|
||||
text-slate-200 focus:outline-none focus:border-blue-500"
|
||||
>
|
||||
{p.options.map(o => <option key={o} value={o}>{o}</option>)}
|
||||
</select>
|
||||
) : (
|
||||
<input
|
||||
type={p.type}
|
||||
value={values[p.name]}
|
||||
onChange={e => setValue(p.name, e.target.value)}
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-1.5 text-xs
|
||||
text-slate-200 focus:outline-none focus:border-blue-500"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<LogViewer lines={logs} />
|
||||
|
||||
<button
|
||||
onClick={run}
|
||||
disabled={!connected || status === 'running'}
|
||||
className="flex items-center gap-2 px-4 py-2 rounded-lg text-xs font-medium
|
||||
bg-blue-600 hover:bg-blue-500 disabled:opacity-40 disabled:cursor-not-allowed transition-colors"
|
||||
>
|
||||
{status === 'running'
|
||||
? <><Loader2 size={13} className="animate-spin"/> Running…</>
|
||||
: <><Play size={13}/> Run {stage.label}</>
|
||||
}
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default function PipelineRunner({ connected }) {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
{!connected && (
|
||||
<div className="rounded-xl border border-yellow-700/40 bg-yellow-900/10 px-4 py-3 text-xs text-yellow-400">
|
||||
Connect to the SSH server first to run pipeline stages.
|
||||
</div>
|
||||
)}
|
||||
{STAGES.map(s => (
|
||||
<StageCard key={s.id} stage={s} connected={connected} />
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
201
frontend/src/components/QAPairViewer.jsx
Normal file
201
frontend/src/components/QAPairViewer.jsx
Normal file
@@ -0,0 +1,201 @@
|
||||
import React, { useState, useEffect } from 'react'
|
||||
import axios from 'axios'
|
||||
import { RefreshCw, Search, Edit3, X, Check, ChevronDown } from 'lucide-react'
|
||||
|
||||
const STAGES = ['generated', 'curated', 'final']
|
||||
|
||||
export default function QAPairViewer({ connected }) {
|
||||
const [stage, setStage] = useState('generated')
|
||||
const [files, setFiles] = useState([])
|
||||
const [file, setFile] = useState('')
|
||||
const [pairs, setPairs] = useState([])
|
||||
const [filtered, setFiltered] = useState([])
|
||||
const [search, setSearch] = useState('')
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [editing, setEditing] = useState(null) // { index, question, answer }
|
||||
const [error, setError] = useState('')
|
||||
|
||||
useEffect(() => {
|
||||
if (!connected) return
|
||||
axios.get(`/api/files/${stage}`)
|
||||
.then(({ data }) => {
|
||||
setFiles(data.files.map(f => f.name))
|
||||
setFile(data.files[0]?.name || '')
|
||||
})
|
||||
.catch(() => setFiles([]))
|
||||
}, [stage, connected])
|
||||
|
||||
useEffect(() => {
|
||||
if (!file || !connected) return
|
||||
setLoading(true); setError('')
|
||||
axios.get(`/api/pairs/${file}?stage=${stage}`)
|
||||
.then(({ data }) => { setPairs(data.pairs); setFiltered(data.pairs) })
|
||||
.catch(err => setError(err.response?.data?.detail || err.message))
|
||||
.finally(() => setLoading(false))
|
||||
}, [file, stage, connected])
|
||||
|
||||
useEffect(() => {
|
||||
if (!search.trim()) { setFiltered(pairs); return }
|
||||
const q = search.toLowerCase()
|
||||
setFiltered(pairs.filter(p =>
|
||||
JSON.stringify(p).toLowerCase().includes(q)
|
||||
))
|
||||
}, [search, pairs])
|
||||
|
||||
const saveEdit = () => {
|
||||
if (!editing) return
|
||||
const updated = [...pairs]
|
||||
updated[editing.index] = { ...updated[editing.index], ...editing }
|
||||
setPairs(updated)
|
||||
setFiltered(updated)
|
||||
setEditing(null)
|
||||
}
|
||||
|
||||
const scoreColor = score => {
|
||||
if (!score && score !== 0) return 'text-slate-500'
|
||||
if (score >= 8) return 'text-green-400'
|
||||
if (score >= 5) return 'text-yellow-400'
|
||||
return 'text-red-400'
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
|
||||
{/* Controls */}
|
||||
<div className="flex flex-wrap gap-3 items-center">
|
||||
<div className="flex gap-1 bg-[#161b27] rounded-xl p-1">
|
||||
{STAGES.map(s => (
|
||||
<button key={s} onClick={() => setStage(s)}
|
||||
className={`px-3 py-1 rounded-lg text-xs font-medium transition-colors capitalize
|
||||
${stage === s ? 'bg-blue-600 text-white' : 'text-slate-400 hover:text-slate-200'}`}>
|
||||
{s}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="relative">
|
||||
<ChevronDown size={12} className="absolute right-3 top-2.5 text-slate-500 pointer-events-none"/>
|
||||
<select
|
||||
value={file}
|
||||
onChange={e => setFile(e.target.value)}
|
||||
className="bg-[#161b27] border border-slate-700 rounded-lg px-3 py-1.5 pr-8 text-xs
|
||||
text-slate-200 focus:outline-none focus:border-blue-500 appearance-none"
|
||||
>
|
||||
{files.map(f => <option key={f} value={f}>{f}</option>)}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div className="relative flex-1 min-w-48">
|
||||
<Search size={13} className="absolute left-3 top-2 text-slate-500"/>
|
||||
<input
|
||||
value={search}
|
||||
onChange={e => setSearch(e.target.value)}
|
||||
placeholder="Search pairs…"
|
||||
className="w-full bg-[#161b27] border border-slate-700 rounded-lg pl-8 pr-3 py-1.5 text-xs
|
||||
text-slate-200 focus:outline-none focus:border-blue-500"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<button onClick={() => setFile(f => f)}
|
||||
className="flex items-center gap-1 text-xs text-slate-500 hover:text-slate-300 transition-colors">
|
||||
<RefreshCw size={12}/> Reload
|
||||
</button>
|
||||
|
||||
<span className="text-xs text-slate-500">{filtered.length} pairs</span>
|
||||
</div>
|
||||
|
||||
{error && <p className="text-xs text-red-400">{error}</p>}
|
||||
|
||||
{/* Table */}
|
||||
<div className="rounded-xl border border-slate-700/50 bg-[#161b27] overflow-hidden">
|
||||
{loading ? (
|
||||
<p className="py-10 text-center text-xs text-slate-500">Loading…</p>
|
||||
) : filtered.length === 0 ? (
|
||||
<p className="py-10 text-center text-xs text-slate-500">No pairs found</p>
|
||||
) : (
|
||||
<div className="overflow-auto max-h-[60vh]">
|
||||
<table className="w-full text-xs">
|
||||
<thead className="sticky top-0 bg-[#161b27] border-b border-slate-700/40">
|
||||
<tr className="text-slate-500">
|
||||
<th className="text-left px-4 py-2 w-8">#</th>
|
||||
<th className="text-left px-4 py-2 w-1/2">Question</th>
|
||||
<th className="text-left px-4 py-2">Answer</th>
|
||||
<th className="text-right px-4 py-2 w-16">Score</th>
|
||||
<th className="px-4 py-2 w-8"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{filtered.map((pair, i) => (
|
||||
<tr key={i} className="border-b border-slate-700/20 hover:bg-slate-700/10 align-top">
|
||||
<td className="px-4 py-2 text-slate-600">{i + 1}</td>
|
||||
<td className="px-4 py-2 text-slate-300 max-w-xs">
|
||||
<p className="line-clamp-3">{pair.question || pair.prompt || JSON.stringify(pair)}</p>
|
||||
</td>
|
||||
<td className="px-4 py-2 text-slate-400">
|
||||
<p className="line-clamp-3">{pair.answer || pair.response || pair.output || '—'}</p>
|
||||
</td>
|
||||
<td className={`px-4 py-2 text-right font-medium ${scoreColor(pair.score)}`}>
|
||||
{pair.score != null ? pair.score : '—'}
|
||||
</td>
|
||||
<td className="px-4 py-2">
|
||||
<button
|
||||
onClick={() => setEditing({ index: i, ...pair })}
|
||||
className="p-1 rounded hover:bg-blue-600/20 text-slate-500 hover:text-blue-400 transition-colors"
|
||||
>
|
||||
<Edit3 size={12}/>
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Edit modal */}
|
||||
{editing && (
|
||||
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/60 p-6">
|
||||
<div className="bg-[#161b27] border border-slate-700 rounded-xl w-full max-w-2xl space-y-4 p-6">
|
||||
<div className="flex items-center justify-between">
|
||||
<h3 className="text-sm font-semibold text-slate-200">Edit Pair #{editing.index + 1}</h3>
|
||||
<button onClick={() => setEditing(null)} className="text-slate-500 hover:text-slate-200">
|
||||
<X size={16}/>
|
||||
</button>
|
||||
</div>
|
||||
<div>
|
||||
<label className="block text-xs text-slate-500 mb-1">Question</label>
|
||||
<textarea
|
||||
rows={3}
|
||||
value={editing.question || editing.prompt || ''}
|
||||
onChange={e => setEditing(v => ({ ...v, question: e.target.value }))}
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-2 text-xs
|
||||
text-slate-200 focus:outline-none focus:border-blue-500 resize-none"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label className="block text-xs text-slate-500 mb-1">Answer</label>
|
||||
<textarea
|
||||
rows={5}
|
||||
value={editing.answer || editing.response || editing.output || ''}
|
||||
onChange={e => setEditing(v => ({ ...v, answer: e.target.value }))}
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-2 text-xs
|
||||
text-slate-200 focus:outline-none focus:border-blue-500 resize-none"
|
||||
/>
|
||||
</div>
|
||||
<div className="flex justify-end gap-2">
|
||||
<button onClick={() => setEditing(null)}
|
||||
className="px-4 py-2 rounded-lg text-xs text-slate-400 hover:text-slate-200 transition-colors">
|
||||
Cancel
|
||||
</button>
|
||||
<button onClick={saveEdit}
|
||||
className="flex items-center gap-2 px-4 py-2 rounded-lg text-xs bg-blue-600 hover:bg-blue-500 transition-colors">
|
||||
<Check size={13}/> Save
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
146
frontend/src/components/Terminal.jsx
Normal file
146
frontend/src/components/Terminal.jsx
Normal file
@@ -0,0 +1,146 @@
|
||||
import React, { useEffect, useRef, useState } from 'react'
|
||||
import { Terminal as XTerm } from '@xterm/xterm'
|
||||
import { FitAddon } from '@xterm/addon-fit'
|
||||
import { WebLinksAddon } from '@xterm/addon-web-links'
|
||||
import '@xterm/xterm/css/xterm.css'
|
||||
import { TerminalSquare, RefreshCw } from 'lucide-react'
|
||||
|
||||
const WS_BASE = `ws://${window.location.host}`
|
||||
|
||||
export default function Terminal({ connected }) {
|
||||
const containerRef = useRef(null)
|
||||
const termRef = useRef(null)
|
||||
const fitRef = useRef(null)
|
||||
const wsRef = useRef(null)
|
||||
const [status, setStatus] = useState('idle') // idle | open | closed | error
|
||||
|
||||
const boot = () => {
|
||||
// Clean up existing
|
||||
if (wsRef.current) wsRef.current.close()
|
||||
if (termRef.current) { termRef.current.dispose(); termRef.current = null }
|
||||
|
||||
if (!containerRef.current) return
|
||||
|
||||
const term = new XTerm({
|
||||
theme: {
|
||||
background: '#0a0d14',
|
||||
foreground: '#e2e8f0',
|
||||
cursor: '#60a5fa',
|
||||
cursorAccent: '#0a0d14',
|
||||
black: '#1e2130',
|
||||
red: '#f87171',
|
||||
green: '#34d399',
|
||||
yellow: '#fbbf24',
|
||||
blue: '#60a5fa',
|
||||
magenta: '#a78bfa',
|
||||
cyan: '#22d3ee',
|
||||
white: '#e2e8f0',
|
||||
},
|
||||
fontSize: 13,
|
||||
fontFamily: "'JetBrains Mono', 'Fira Code', Consolas, monospace",
|
||||
cursorBlink: true,
|
||||
scrollback: 5000,
|
||||
})
|
||||
|
||||
const fit = new FitAddon()
|
||||
term.loadAddon(fit)
|
||||
term.loadAddon(new WebLinksAddon())
|
||||
term.open(containerRef.current)
|
||||
fit.fit()
|
||||
|
||||
termRef.current = term
|
||||
fitRef.current = fit
|
||||
|
||||
const ws = new WebSocket(`${WS_BASE}/api/terminal`)
|
||||
wsRef.current = ws
|
||||
|
||||
ws.binaryType = 'arraybuffer'
|
||||
|
||||
ws.onopen = () => {
|
||||
setStatus('open')
|
||||
term.writeln('\x1b[1;34mSSH Terminal — connected\x1b[0m')
|
||||
}
|
||||
|
||||
ws.onmessage = e => {
|
||||
const data = e.data instanceof ArrayBuffer
|
||||
? new Uint8Array(e.data)
|
||||
: e.data
|
||||
term.write(data)
|
||||
}
|
||||
|
||||
ws.onclose = () => {
|
||||
setStatus('closed')
|
||||
term.writeln('\r\n\x1b[1;31m[Session closed]\x1b[0m')
|
||||
}
|
||||
|
||||
ws.onerror = () => {
|
||||
setStatus('error')
|
||||
term.writeln('\r\n\x1b[1;31m[WebSocket error]\x1b[0m')
|
||||
}
|
||||
|
||||
term.onData(data => {
|
||||
if (ws.readyState === WebSocket.OPEN) ws.send(data)
|
||||
})
|
||||
|
||||
term.onResize(({ cols, rows }) => {
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(JSON.stringify({ type: 'resize', cols, rows }))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Auto-boot when connected
|
||||
useEffect(() => {
|
||||
if (connected) boot()
|
||||
return () => {
|
||||
wsRef.current?.close()
|
||||
termRef.current?.dispose()
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [connected])
|
||||
|
||||
// Resize observer
|
||||
useEffect(() => {
|
||||
const obs = new ResizeObserver(() => fitRef.current?.fit())
|
||||
if (containerRef.current) obs.observe(containerRef.current)
|
||||
return () => obs.disconnect()
|
||||
}, [])
|
||||
|
||||
const statusColor = {
|
||||
idle: 'text-slate-500',
|
||||
open: 'text-green-400',
|
||||
closed: 'text-yellow-400',
|
||||
error: 'text-red-400',
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex flex-col h-full space-y-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className={`flex items-center gap-2 text-xs ${statusColor[status]}`}>
|
||||
<TerminalSquare size={14}/>
|
||||
SSH Terminal — {status}
|
||||
</div>
|
||||
<button
|
||||
onClick={boot}
|
||||
disabled={!connected}
|
||||
className="flex items-center gap-1 text-xs text-slate-500 hover:text-slate-300
|
||||
disabled:opacity-40 disabled:cursor-not-allowed transition-colors"
|
||||
>
|
||||
<RefreshCw size={12}/> Reconnect
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{!connected ? (
|
||||
<div className="flex-1 flex items-center justify-center rounded-xl border border-slate-700/50 bg-[#0a0d14]">
|
||||
<p className="text-xs text-slate-600">Connect to SSH server to use the terminal</p>
|
||||
</div>
|
||||
) : (
|
||||
<div
|
||||
ref={containerRef}
|
||||
className="flex-1 rounded-xl overflow-hidden border border-slate-700/50"
|
||||
style={{ minHeight: '480px', background: '#0a0d14' }}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
265
frontend/src/components/TrainingMonitor.jsx
Normal file
265
frontend/src/components/TrainingMonitor.jsx
Normal file
@@ -0,0 +1,265 @@
|
||||
import React, { useState, useEffect, useRef } from 'react'
|
||||
import axios from 'axios'
|
||||
import {
|
||||
LineChart, Line, XAxis, YAxis, CartesianGrid, Tooltip,
|
||||
ResponsiveContainer, Legend,
|
||||
} from 'recharts'
|
||||
import { Play, Square, RefreshCw, Cpu, Thermometer, Zap } from 'lucide-react'
|
||||
|
||||
const WS_BASE = `ws://${window.location.host}`
|
||||
|
||||
function GpuCard({ gpu }) {
|
||||
const memPct = Math.round((gpu.memory_used / gpu.memory_total) * 100)
|
||||
return (
|
||||
<div className="rounded-xl border border-slate-700/50 bg-[#161b27] p-4 space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
<Cpu size={15} className="text-purple-400"/>
|
||||
<span className="text-xs font-semibold text-slate-200 truncate max-w-[180px]">{gpu.name}</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-1 text-xs text-orange-400">
|
||||
<Thermometer size={12}/>{gpu.temperature}°C
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* GPU utilization */}
|
||||
<div>
|
||||
<div className="flex justify-between text-xs mb-1">
|
||||
<span className="text-slate-500">GPU Utilization</span>
|
||||
<span className="text-purple-400 font-medium">{gpu.utilization}%</span>
|
||||
</div>
|
||||
<div className="h-1.5 bg-slate-700 rounded-full overflow-hidden">
|
||||
<div className="h-full bg-purple-500 rounded-full transition-all"
|
||||
style={{ width: `${gpu.utilization}%` }}/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* VRAM */}
|
||||
<div>
|
||||
<div className="flex justify-between text-xs mb-1">
|
||||
<span className="text-slate-500">VRAM</span>
|
||||
<span className="text-blue-400 font-medium">{gpu.memory_used}/{gpu.memory_total} MB ({memPct}%)</span>
|
||||
</div>
|
||||
<div className="h-1.5 bg-slate-700 rounded-full overflow-hidden">
|
||||
<div className="h-full rounded-full transition-all"
|
||||
style={{
|
||||
width: `${memPct}%`,
|
||||
background: memPct > 90 ? '#ef4444' : memPct > 70 ? '#f59e0b' : '#3b82f6'
|
||||
}}/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{gpu.power_draw && (
|
||||
<div className="flex items-center gap-1 text-xs text-yellow-400">
|
||||
<Zap size={11}/> {gpu.power_draw.toFixed(0)} W
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default function TrainingMonitor({ connected }) {
|
||||
const [files, setFiles] = useState([])
|
||||
const [config, setConfig] = useState({
|
||||
model_name: 'llama3.1:8b',
|
||||
dataset_path: '',
|
||||
output_dir: '/opt/synthetic/output',
|
||||
num_epochs: 3,
|
||||
batch_size: 2,
|
||||
learning_rate: 0.0002,
|
||||
})
|
||||
const [gpus, setGpus] = useState([])
|
||||
const [logs, setLogs] = useState([])
|
||||
const [lossData, setLossData] = useState([])
|
||||
const [running, setRunning] = useState(false)
|
||||
const wsRef = useRef(null)
|
||||
const logRef = useRef(null)
|
||||
|
||||
// Poll GPU stats
|
||||
useEffect(() => {
|
||||
if (!connected) return
|
||||
const poll = async () => {
|
||||
try {
|
||||
const { data } = await axios.get('/api/gpu')
|
||||
setGpus(data.gpus || [])
|
||||
} catch {}
|
||||
}
|
||||
poll()
|
||||
const id = setInterval(poll, 5000)
|
||||
return () => clearInterval(id)
|
||||
}, [connected])
|
||||
|
||||
// Load final files for dataset selector
|
||||
useEffect(() => {
|
||||
if (!connected) return
|
||||
axios.get('/api/files/final')
|
||||
.then(({ data }) => setFiles(data.files.map(f => f.name)))
|
||||
.catch(() => {})
|
||||
}, [connected])
|
||||
|
||||
useEffect(() => {
|
||||
logRef.current?.scrollTo(0, logRef.current.scrollHeight)
|
||||
}, [logs])
|
||||
|
||||
const parseLoss = line => {
|
||||
const m = line.match(/loss[:\s=]+([0-9.]+)/i)
|
||||
const s = line.match(/step[:\s=]+(\d+)/i)
|
||||
if (m) {
|
||||
setLossData(d => [...d, {
|
||||
step: s ? parseInt(s[1]) : d.length + 1,
|
||||
loss: parseFloat(m[1]),
|
||||
}])
|
||||
}
|
||||
}
|
||||
|
||||
const startTraining = () => {
|
||||
if (wsRef.current) wsRef.current.close()
|
||||
setLogs([]); setLossData([]); setRunning(true)
|
||||
|
||||
const params = new URLSearchParams({
|
||||
model_name: config.model_name,
|
||||
dataset_path: config.dataset_path,
|
||||
output_dir: config.output_dir,
|
||||
num_epochs: config.num_epochs,
|
||||
batch_size: config.batch_size,
|
||||
learning_rate: config.learning_rate,
|
||||
})
|
||||
const ws = new WebSocket(`${WS_BASE}/api/train?${params}`)
|
||||
wsRef.current = ws
|
||||
|
||||
ws.onmessage = e => {
|
||||
const msg = JSON.parse(e.data)
|
||||
if (msg.type === 'log') {
|
||||
setLogs(l => [...l, msg.data])
|
||||
parseLoss(msg.data)
|
||||
}
|
||||
if (msg.type === 'done' || msg.type === 'error') {
|
||||
setRunning(false)
|
||||
ws.close()
|
||||
}
|
||||
}
|
||||
ws.onerror = () => setRunning(false)
|
||||
}
|
||||
|
||||
const stopTraining = () => {
|
||||
wsRef.current?.close()
|
||||
setRunning(false)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
|
||||
{/* GPU Cards */}
|
||||
<div className="space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<h3 className="text-xs font-semibold text-slate-400 uppercase tracking-wider">GPU Status</h3>
|
||||
<button onClick={() => connected && axios.get('/api/gpu').then(r => setGpus(r.data.gpus))}
|
||||
className="flex items-center gap-1 text-xs text-slate-500 hover:text-slate-300 transition-colors">
|
||||
<RefreshCw size={11}/> Refresh
|
||||
</button>
|
||||
</div>
|
||||
{gpus.length === 0
|
||||
? <p className="text-xs text-slate-600 py-4 text-center">
|
||||
{connected ? 'No GPU data available' : 'Connect to view GPU stats'}
|
||||
</p>
|
||||
: gpus.map((g, i) => <GpuCard key={i} gpu={g}/>)
|
||||
}
|
||||
</div>
|
||||
|
||||
{/* Training config */}
|
||||
<div className="rounded-xl border border-slate-700/50 bg-[#161b27] p-4 space-y-3">
|
||||
<h3 className="text-xs font-semibold text-slate-400 uppercase tracking-wider">Training Config</h3>
|
||||
|
||||
{[
|
||||
{ key: 'model_name', label: 'Base Model', type: 'text' },
|
||||
{ key: 'output_dir', label: 'Output Dir', type: 'text' },
|
||||
{ key: 'num_epochs', label: 'Epochs', type: 'number' },
|
||||
{ key: 'batch_size', label: 'Batch Size', type: 'number' },
|
||||
{ key: 'learning_rate',label: 'Learning Rate', type: 'number', step: '0.00001' },
|
||||
].map(f => (
|
||||
<div key={f.key}>
|
||||
<label className="block text-xs text-slate-500 mb-1">{f.label}</label>
|
||||
<input
|
||||
type={f.type}
|
||||
step={f.step}
|
||||
value={config[f.key]}
|
||||
onChange={e => setConfig(c => ({ ...c, [f.key]: e.target.value }))}
|
||||
disabled={running}
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-1.5 text-xs
|
||||
text-slate-200 focus:outline-none focus:border-blue-500 disabled:opacity-50"
|
||||
/>
|
||||
</div>
|
||||
))}
|
||||
|
||||
<div>
|
||||
<label className="block text-xs text-slate-500 mb-1">Dataset File</label>
|
||||
<select
|
||||
value={config.dataset_path}
|
||||
onChange={e => setConfig(c => ({ ...c, dataset_path: e.target.value }))}
|
||||
disabled={running}
|
||||
className="w-full bg-[#0f1117] border border-slate-700 rounded-lg px-3 py-1.5 text-xs
|
||||
text-slate-200 focus:outline-none focus:border-blue-500 disabled:opacity-50"
|
||||
>
|
||||
<option value="">Select dataset…</option>
|
||||
{files.map(f => <option key={f} value={`/opt/synthetic/synthetic-data-kit/data/final/${f}`}>{f}</option>)}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div className="flex gap-2 pt-1">
|
||||
<button
|
||||
onClick={startTraining}
|
||||
disabled={!connected || running || !config.dataset_path}
|
||||
className="flex-1 flex items-center justify-center gap-2 py-2 rounded-lg text-xs font-medium
|
||||
bg-blue-600 hover:bg-blue-500 disabled:opacity-40 disabled:cursor-not-allowed transition-colors"
|
||||
>
|
||||
<Play size={13}/> Start Training
|
||||
</button>
|
||||
{running && (
|
||||
<button
|
||||
onClick={stopTraining}
|
||||
className="flex items-center gap-2 px-3 py-2 rounded-lg text-xs font-medium
|
||||
bg-red-600/20 border border-red-700/40 text-red-400 hover:bg-red-600/30 transition-colors"
|
||||
>
|
||||
<Square size={13}/> Stop
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Loss chart */}
|
||||
{lossData.length > 1 && (
|
||||
<div className="rounded-xl border border-slate-700/50 bg-[#161b27] p-4">
|
||||
<h3 className="text-xs font-semibold text-slate-400 uppercase tracking-wider mb-3">Training Loss</h3>
|
||||
<ResponsiveContainer width="100%" height={200}>
|
||||
<LineChart data={lossData}>
|
||||
<CartesianGrid strokeDasharray="3 3" stroke="#2d3348"/>
|
||||
<XAxis dataKey="step" tick={{ fill: '#64748b', fontSize: 10 }} label={{ value: 'Step', fill: '#64748b', fontSize: 10, position: 'insideBottom', offset: -5 }}/>
|
||||
<YAxis tick={{ fill: '#64748b', fontSize: 10 }}/>
|
||||
<Tooltip contentStyle={{ background: '#161b27', border: '1px solid #334155', borderRadius: '8px', fontSize: '12px' }}/>
|
||||
<Line type="monotone" dataKey="loss" stroke="#3b82f6" strokeWidth={2} dot={false}/>
|
||||
</LineChart>
|
||||
</ResponsiveContainer>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Log output */}
|
||||
<div className="rounded-xl border border-slate-700/50 bg-[#161b27] overflow-hidden">
|
||||
<div className="px-4 py-2 border-b border-slate-700/50 text-xs text-slate-500">Training Log</div>
|
||||
<div ref={logRef} className="h-48 overflow-y-auto bg-[#0a0d14] p-3 font-mono text-xs leading-relaxed">
|
||||
{logs.length === 0
|
||||
? <span className="text-slate-600">No training output yet…</span>
|
||||
: logs.map((l, i) => (
|
||||
<div key={i} className={
|
||||
l.includes('Error') ? 'text-red-400' :
|
||||
l.includes('loss') ? 'text-blue-300' : 'text-slate-300'
|
||||
}>{l}</div>
|
||||
))
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
41
frontend/src/index.css
Normal file
41
frontend/src/index.css
Normal file
@@ -0,0 +1,41 @@
|
||||
@tailwind base;
|
||||
@tailwind components;
|
||||
@tailwind utilities;
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
background-color: #0f1117;
|
||||
color: #e2e8f0;
|
||||
font-family: 'JetBrains Mono', 'Fira Code', Consolas, monospace;
|
||||
}
|
||||
|
||||
/* xterm.js overrides */
|
||||
.xterm-viewport {
|
||||
overflow-y: auto !important;
|
||||
}
|
||||
|
||||
/* Custom scrollbar */
|
||||
::-webkit-scrollbar {
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
}
|
||||
::-webkit-scrollbar-track {
|
||||
background: #1e2130;
|
||||
}
|
||||
::-webkit-scrollbar-thumb {
|
||||
background: #3b4563;
|
||||
border-radius: 3px;
|
||||
}
|
||||
::-webkit-scrollbar-thumb:hover {
|
||||
background: #4f5e80;
|
||||
}
|
||||
|
||||
/* Log viewer */
|
||||
.log-line-error { color: #f87171; }
|
||||
.log-line-warning { color: #fbbf24; }
|
||||
.log-line-success { color: #34d399; }
|
||||
.log-line-info { color: #60a5fa; }
|
||||
10
frontend/src/index.jsx
Normal file
10
frontend/src/index.jsx
Normal file
@@ -0,0 +1,10 @@
|
||||
import React from 'react'
|
||||
import ReactDOM from 'react-dom/client'
|
||||
import App from './App'
|
||||
import './index.css'
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')).render(
|
||||
<React.StrictMode>
|
||||
<App />
|
||||
</React.StrictMode>
|
||||
)
|
||||
12
frontend/tailwind.config.js
Normal file
12
frontend/tailwind.config.js
Normal file
@@ -0,0 +1,12 @@
|
||||
/** @type {import('tailwindcss').Config} */
|
||||
export default {
|
||||
content: ['./index.html', './src/**/*.{js,jsx}'],
|
||||
theme: {
|
||||
extend: {
|
||||
fontFamily: {
|
||||
mono: ['JetBrains Mono', 'Fira Code', 'Consolas', 'monospace'],
|
||||
},
|
||||
},
|
||||
},
|
||||
plugins: [],
|
||||
}
|
||||
16
frontend/vite.config.js
Normal file
16
frontend/vite.config.js
Normal file
@@ -0,0 +1,16 @@
|
||||
import { defineConfig } from 'vite'
|
||||
import react from '@vitejs/plugin-react'
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
server: {
|
||||
port: 3000,
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://localhost:8080',
|
||||
changeOrigin: true,
|
||||
ws: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
Reference in New Issue
Block a user