mirror of
https://github.com/lukaszraczylo/graphql-monitoring-proxy.git
synced 2026-06-05 23:03:48 +00:00
lukaszraczylo
e3e9f7d181
* General improvements and bug fixes. * Improve tests coverage. * fixup! Improve tests coverage. * Update README.md with latest changes. * Fix the uint32 * Resolve issue with race condition for logging. * fixup! Merge remote-tracking branch 'origin/main' into improvements-mid-apr-2025 * Fix the test of the rate limiter * Add default ratelimit.json file * Update dependencies. * Significant refactor. * fixup! Significant refactor. * fixup! Merge remote-tracking branch 'origin/main' into improvements-mid-apr-2025 * fixup! fixup! Merge remote-tracking branch 'origin/main' into improvements-mid-apr-2025 * fixup! fixup! fixup! Merge remote-tracking branch 'origin/main' into improvements-mid-apr-2025 * fixup! fixup! fixup! fixup! fixup! Merge remote-tracking branch 'origin/main' into improvements-mid-apr-2025 * fixup! fixup! fixup! fixup! fixup! fixup! Merge remote-tracking branch 'origin/main' into improvements-mid-apr-2025 * fixup! fixup! fixup! fixup! fixup! fixup! fixup! Merge remote-tracking branch 'origin/main' into improvements-mid-apr-2025 * fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Merge remote-tracking branch 'origin/main' into improvements-mid-apr-2025 * fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Merge remote-tracking branch 'origin/main' into improvements-mid-apr-2025 * fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Merge remote-tracking branch 'origin/main' into improvements-mid-apr-2025 * fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Merge remote-tracking branch 'origin/main' into improvements-mid-apr-2025 * fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Merge remote-tracking branch 'origin/main' into improvements-mid-apr-2025 * Enhance admin dashboard with real-time WebSocket streaming and charts Dashboard improvements: - Added Chart.js for time-series visualization - Created real-time graphs for RPS and cache hit rate - Added new statistics displays: * System uptime with human-readable format * Total requests with success/failure breakdown * Current and average RPS * Success rate with progress bars * Cache hit rate and memory usage with visual indicators * Detailed cache statistics - Implemented WebSocket streaming endpoint (/admin/ws/stats) - Real-time updates every 2 seconds via WebSocket - Automatic fallback to polling if WebSocket unavailable - Connection status indicator - Progress bars for success rate, cache hit rate, and memory usage Backend enhancements: - New WebSocket handler for streaming all statistics - gatherAllStats() method to collect comprehensive metrics - Streams data every 2 seconds to connected clients - Automatic reconnection handling - Maintains up to 60 data points per chart The dashboard now provides comprehensive real-time monitoring with: - Live metrics streaming - Historical trend visualization - Responsive design with visual indicators - Graceful degradation to polling mode 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Fix WebSocket message handling for real-time stats streaming Issues fixed: - Removed blocking default case that prevented ticker from firing - Separated read and write operations into proper goroutines - Added proper ping/pong handlers with read deadlines - Implemented done channel for clean disconnection signaling - Send initial stats immediately on connection The WebSocket now properly: - Streams stats every 2 seconds via ticker - Handles client disconnections gracefully - Maintains connection with ping/pong - Detects connection drops via read goroutine - Non-blocking message handling 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Add Redis-based cluster mode for distributed metrics aggregation When using Redis for caching, proxies now automatically form a cluster and aggregate metrics across all instances for unified monitoring. Features: - Metrics Aggregator: Publishes instance metrics to Redis every 5s - Cluster Mode API: /admin/api/cluster/stats and /admin/api/cluster/instances - Dashboard Cluster View: Toggle between single instance and cluster view - Auto-discovery: Detects cluster mode automatically via Redis - Instance Management: Each instance gets unique ID (hostname + UUID) - Graceful Cleanup: Removes metrics from Redis on shutdown - TTL-based expiration: Stale instances auto-expire after 30s Cluster metrics include: - Aggregated requests (total, succeeded, failed, success rate) - Combined RPS across all instances - Total cache hits/misses with cluster-wide hit rate - Per-instance health status and uptime - Active connections and WebSocket stats - Request coalescing backend savings Dashboard improvements: - Cluster Status section showing total/healthy instances - Instance Details section with per-node metrics - Cluster View toggle in header - Automatic detection of cluster availability - Individual instance cards with health indicators - "Current" badge for the instance you're viewing Architecture: - Uses Redis SET to track active instances - Each instance publishes to redis key: graphql-proxy:metrics:instances:{id} - 30s TTL ensures stale instances are removed - Aggregator started automatically when Redis cache enabled - Registered with shutdown manager for graceful cleanup Environment: Automatically enabled when ENABLE_REDIS_CACHE=true 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Fix cluster instance details using correct JSON field names JavaScript was using Go struct field names (PascalCase) instead of JSON field names (snake_case), causing all instance metrics to show as 0 or undefined. Fixed references: - instance.InstanceID → instance.instance_id - instance.Hostname → instance.hostname - instance.UptimeSeconds → instance.uptime_seconds - instance.Stats → instance.stats - instance.Health → instance.health Also added fallback to check both instance.cache_summary and stats.cache_summary for better compatibility. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Enhance Cluster View toggle visibility and styling Improvements: - Replaced basic checkbox with custom toggle switch - Added prominent card-style container with backdrop blur - Positioned toggle in header next to dashboard title - Toggle switch with smooth animation (slides left/right) - Green color when enabled (#10b981) - Hover effects with slight lift - Better typography with font weights and spacing - Info text positioned below toggle label - Disabled state with reduced opacity - Responsive layout with flexbox The toggle is now much more visible and professional-looking, making it clear when cluster mode is available and active. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Add comprehensive debugging for cluster mode metrics issues Backend improvements: - Fixed metrics structure: ensure Stats always has correct inner structure - Added defensive nil checks for instance.Stats in aggregation - Added debug logging in publishMetrics to verify data being sent - Added warning logging in aggregateStats when data is missing - Log actual keys present in Stats when 'requests' is missing - Initialize empty maps instead of leaving fields nil Frontend improvements: - Added console.log statements to trace cluster data flow - Log cluster data structure on receive - Log stats keys and structure - Log instances array and count - Warn when expected data is missing - Added fallback values (|| 0) for display fields This will help diagnose why cluster view shows zeros by logging: 1. What data is being published to Redis 2. What data is being retrieved from Redis 3. What structure the data has at each step 4. What keys are present vs expected Check browser console and server logs to see where the data flow breaks. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Add cluster debug endpoint for troubleshooting New endpoint: GET /admin/api/cluster/debug Returns comprehensive debug information: - Whether metrics aggregator is initialized - Redis cache enabled status - Current instance ID - Cluster mode status - Total/healthy instance counts - Sample instance structure with keys - Sample requests data structure - Error messages if any This helps diagnose cluster mode issues by showing: 1. If Redis is actually enabled 2. If aggregator is initialized 3. What data structure is being stored 4. What keys are present in Stats 5. Sample of actual data being aggregated Visit http://localhost:8181/admin/api/cluster/debug to see what's happening under the hood. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Fix cluster mode initialization and improve Redis error visibility Critical fixes: 1. Move metrics aggregator initialization BEFORE cache initialization - Runs independently even if CacheEnable=false - Only requires CacheRedisEnable=true - This was causing aggregator to not initialize when cache was disabled 2. Promote Redis errors from Warning to Error level - Changed "Failed to publish" from Warning to Error with ❌ CRITICAL prefix - Added detailed error context (instance_id, keys, error message) - Added success logging with ✓ confirmation - Log command count and data size on success 3. Enhanced startup logging - Log "Initializing metrics aggregator" with Redis URL/DB - Log "✓ Successfully initialized" with instance ID - Log "FAILED to initialize" as ERROR (was Warning) Why this matters: - If Redis cache is disabled but Redis is available, cluster mode should still work - Previous code only initialized aggregator inside cache initialization block - Redis publish errors were being silently logged as warnings - No visibility into whether metrics were actually being stored After this fix: - Cluster mode works even with ENABLE_GLOBAL_CACHE=false + ENABLE_REDIS_CACHE=true - Redis errors are immediately visible in logs - Clear success/failure indicators - Better troubleshooting information 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Add force-publish endpoint for cluster mode debugging New endpoint: POST /admin/api/cluster/force-publish Forces an immediate metrics publish to Redis and reports results. This helps diagnose why instances aren't appearing: Response includes: - success: true/false - publish_done: confirmation publish was attempted - instances_found: count after publish - error: if retrieval failed - check_logs: reminder to look for log messages Use this to test: curl -X POST http://localhost:8181/admin/api/cluster/force-publish Then check server logs for: ✓ Successfully published metrics to Redis OR ❌ CRITICAL: Failed to publish metrics to Redis This bypasses the 5-second timer and publishes immediately, making it easier to test without waiting. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Fix cluster metrics aggregation and dashboard display issues - Fix metric names: use correct Prometheus metric names (requests_succesful, requests_failed, requests_skipped) - Add automatic stale instance cleanup (>1 minute inactive) - Implement 10-second moving average smoothing for RPS, success rate, and cache hit rate - Add trend indicators (↑ ↗ → ↘ ↓) to show metric direction - Add compact number formatting (1.2M, 3.4K) with full-value tooltips - Add retry budget aggregation (allowed/denied retries, denial rate) - Add circuit breaker aggregation (state counts, per-instance breakdown) - Add coalescing stats aggregation (backend savings percentage) - Fix memory display to show "N/A" for Redis cache (memory tracking not available) - Fix JavaScript error: change hitRate to smoothedHitRate in chart update call - Change Redis operations to use context.Background() instead of parent context - Fix staticcheck warning: omit nil check for map len() This resolves cluster view showing zeros and prevents metrics from disappearing. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Fix cluster instance count not updating in upper right corner The cluster-info element (showing instance count next to "Cluster View" toggle) was only updated during initial page load. Now it updates in real-time whenever cluster stats are received via WebSocket. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Clean up verbose debug logging Removed debug console.log statements from dashboard JavaScript: - WebSocket connection/disconnection logs - Cluster mode availability checks - Cluster stats update debug logs Removed verbose Info logs from Go code that ran frequently: - "Publishing metrics to Redis" (every 5s) - "Metrics gathered successfully" (every 5s) - "Successfully published metrics to Redis" (every 5s) - "Aggregating stats from instances" (frequently) - "Successfully aggregated cluster metrics" (frequently) - "Aggregation complete" (frequently) Kept important logs: - Error and warning logs - Initialization and shutdown logs - Conditional logs (stale instance cleanup, failures) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> --------- Co-authored-by: Claude <noreply@anthropic.com>
1336 lines
49 KiB
HTML
1336 lines
49 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>GraphQL Proxy Admin Dashboard</title>
|
|
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
|
|
<style>
|
|
* {
|
|
margin: 0;
|
|
padding: 0;
|
|
box-sizing: border-box;
|
|
}
|
|
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
|
|
background: #f5f5f5;
|
|
color: #333;
|
|
line-height: 1.6;
|
|
}
|
|
|
|
.container {
|
|
max-width: 1400px;
|
|
margin: 0 auto;
|
|
padding: 20px;
|
|
}
|
|
|
|
header {
|
|
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
|
color: white;
|
|
padding: 30px 0;
|
|
box-shadow: 0 2px 10px rgba(0,0,0,0.1);
|
|
margin-bottom: 30px;
|
|
}
|
|
|
|
h1 {
|
|
font-size: 2em;
|
|
font-weight: 600;
|
|
}
|
|
|
|
.subtitle {
|
|
opacity: 0.9;
|
|
font-size: 0.95em;
|
|
margin-top: 5px;
|
|
}
|
|
|
|
.stats-grid {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
|
|
gap: 20px;
|
|
margin-bottom: 30px;
|
|
}
|
|
|
|
.card {
|
|
background: white;
|
|
border-radius: 12px;
|
|
padding: 24px;
|
|
box-shadow: 0 2px 8px rgba(0,0,0,0.08);
|
|
transition: transform 0.2s, box-shadow 0.2s;
|
|
}
|
|
|
|
.card:hover {
|
|
transform: translateY(-2px);
|
|
box-shadow: 0 4px 12px rgba(0,0,0,0.12);
|
|
}
|
|
|
|
.card-title {
|
|
font-size: 0.85em;
|
|
text-transform: uppercase;
|
|
letter-spacing: 0.5px;
|
|
color: #666;
|
|
margin-bottom: 12px;
|
|
font-weight: 600;
|
|
}
|
|
|
|
.card-value {
|
|
font-size: 2.5em;
|
|
font-weight: 700;
|
|
color: #333;
|
|
line-height: 1;
|
|
}
|
|
|
|
.card-label {
|
|
font-size: 0.9em;
|
|
color: #888;
|
|
margin-top: 8px;
|
|
}
|
|
|
|
.status-indicator {
|
|
display: inline-block;
|
|
width: 12px;
|
|
height: 12px;
|
|
border-radius: 50%;
|
|
margin-right: 8px;
|
|
}
|
|
|
|
.status-healthy {
|
|
background: #10b981;
|
|
box-shadow: 0 0 0 4px rgba(16, 185, 129, 0.2);
|
|
}
|
|
|
|
.status-unhealthy {
|
|
background: #ef4444;
|
|
box-shadow: 0 0 0 4px rgba(239, 68, 68, 0.2);
|
|
}
|
|
|
|
.status-unknown {
|
|
background: #6b7280;
|
|
box-shadow: 0 0 0 4px rgba(107, 114, 128, 0.2);
|
|
}
|
|
|
|
.metric-row {
|
|
display: flex;
|
|
justify-content: space-between;
|
|
padding: 12px 0;
|
|
border-bottom: 1px solid #f0f0f0;
|
|
}
|
|
|
|
.metric-row:last-child {
|
|
border-bottom: none;
|
|
}
|
|
|
|
.metric-label {
|
|
color: #666;
|
|
font-size: 0.95em;
|
|
}
|
|
|
|
.metric-value {
|
|
font-weight: 600;
|
|
color: #333;
|
|
}
|
|
|
|
.btn {
|
|
background: #667eea;
|
|
color: white;
|
|
border: none;
|
|
padding: 10px 20px;
|
|
border-radius: 6px;
|
|
cursor: pointer;
|
|
font-size: 0.9em;
|
|
font-weight: 500;
|
|
transition: background 0.2s;
|
|
}
|
|
|
|
.btn:hover {
|
|
background: #5568d3;
|
|
}
|
|
|
|
.btn:active {
|
|
transform: scale(0.98);
|
|
}
|
|
|
|
.btn-danger {
|
|
background: #ef4444;
|
|
}
|
|
|
|
.btn-danger:hover {
|
|
background: #dc2626;
|
|
}
|
|
|
|
.section-title {
|
|
font-size: 1.5em;
|
|
margin: 40px 0 20px 0;
|
|
color: #333;
|
|
font-weight: 600;
|
|
}
|
|
|
|
.refresh-info {
|
|
text-align: center;
|
|
color: #888;
|
|
font-size: 0.85em;
|
|
margin-top: 30px;
|
|
}
|
|
|
|
.badge {
|
|
display: inline-block;
|
|
padding: 4px 12px;
|
|
border-radius: 12px;
|
|
font-size: 0.8em;
|
|
font-weight: 600;
|
|
}
|
|
|
|
.badge-success {
|
|
background: #d1fae5;
|
|
color: #065f46;
|
|
}
|
|
|
|
.badge-danger {
|
|
background: #fee2e2;
|
|
color: #991b1b;
|
|
}
|
|
|
|
.badge-warning {
|
|
background: #fef3c7;
|
|
color: #92400e;
|
|
}
|
|
|
|
.badge-info {
|
|
background: #dbeafe;
|
|
color: #1e40af;
|
|
}
|
|
|
|
@keyframes pulse {
|
|
0%, 100% {
|
|
opacity: 1;
|
|
}
|
|
50% {
|
|
opacity: 0.5;
|
|
}
|
|
}
|
|
|
|
.loading {
|
|
animation: pulse 2s cubic-bezier(0.4, 0, 0.6, 1) infinite;
|
|
}
|
|
|
|
.chart-container {
|
|
position: relative;
|
|
height: 300px;
|
|
margin-top: 20px;
|
|
}
|
|
|
|
.progress-bar {
|
|
width: 100%;
|
|
height: 8px;
|
|
background: #f0f0f0;
|
|
border-radius: 4px;
|
|
overflow: hidden;
|
|
margin-top: 8px;
|
|
}
|
|
|
|
.progress-fill {
|
|
height: 100%;
|
|
background: linear-gradient(90deg, #667eea 0%, #764ba2 100%);
|
|
transition: width 0.3s ease;
|
|
}
|
|
|
|
.ws-status {
|
|
display: inline-block;
|
|
padding: 4px 8px;
|
|
border-radius: 4px;
|
|
font-size: 0.75em;
|
|
font-weight: 600;
|
|
margin-left: 10px;
|
|
}
|
|
|
|
.ws-connected {
|
|
background: #d1fae5;
|
|
color: #065f46;
|
|
}
|
|
|
|
.ws-disconnected {
|
|
background: #fee2e2;
|
|
color: #991b1b;
|
|
}
|
|
|
|
.cluster-toggle-container {
|
|
background: rgba(255, 255, 255, 0.15);
|
|
padding: 12px 20px;
|
|
border-radius: 8px;
|
|
display: inline-flex;
|
|
align-items: center;
|
|
gap: 12px;
|
|
backdrop-filter: blur(10px);
|
|
border: 1px solid rgba(255, 255, 255, 0.2);
|
|
cursor: pointer;
|
|
transition: all 0.3s ease;
|
|
}
|
|
|
|
.cluster-toggle-container:hover {
|
|
background: rgba(255, 255, 255, 0.25);
|
|
transform: translateY(-1px);
|
|
}
|
|
|
|
.toggle-switch {
|
|
position: relative;
|
|
width: 48px;
|
|
height: 24px;
|
|
background: rgba(255, 255, 255, 0.3);
|
|
border-radius: 12px;
|
|
transition: background 0.3s ease;
|
|
cursor: pointer;
|
|
}
|
|
|
|
.toggle-switch::after {
|
|
content: '';
|
|
position: absolute;
|
|
top: 2px;
|
|
left: 2px;
|
|
width: 20px;
|
|
height: 20px;
|
|
background: white;
|
|
border-radius: 50%;
|
|
transition: transform 0.3s ease;
|
|
box-shadow: 0 2px 4px rgba(0,0,0,0.2);
|
|
}
|
|
|
|
#cluster-mode-toggle {
|
|
display: none;
|
|
}
|
|
|
|
#cluster-mode-toggle:checked + .cluster-toggle-container .toggle-switch {
|
|
background: #10b981;
|
|
}
|
|
|
|
#cluster-mode-toggle:checked + .cluster-toggle-container .toggle-switch::after {
|
|
transform: translateX(24px);
|
|
}
|
|
|
|
#cluster-mode-toggle:disabled + .cluster-toggle-container {
|
|
opacity: 0.5;
|
|
cursor: not-allowed;
|
|
}
|
|
|
|
#cluster-mode-toggle:disabled + .cluster-toggle-container:hover {
|
|
background: rgba(255, 255, 255, 0.15);
|
|
transform: none;
|
|
}
|
|
|
|
.cluster-toggle-label {
|
|
font-size: 0.95em;
|
|
font-weight: 600;
|
|
color: white;
|
|
letter-spacing: 0.3px;
|
|
user-select: none;
|
|
}
|
|
|
|
.cluster-toggle-info {
|
|
font-size: 0.8em;
|
|
opacity: 0.9;
|
|
color: white;
|
|
font-weight: 400;
|
|
}
|
|
|
|
.instance-card {
|
|
background: white;
|
|
border-radius: 12px;
|
|
padding: 20px;
|
|
margin-bottom: 15px;
|
|
box-shadow: 0 2px 8px rgba(0,0,0,0.08);
|
|
}
|
|
|
|
.instance-header {
|
|
display: flex;
|
|
justify-content: space-between;
|
|
align-items: center;
|
|
margin-bottom: 15px;
|
|
padding-bottom: 15px;
|
|
border-bottom: 1px solid #f0f0f0;
|
|
}
|
|
|
|
.instance-title {
|
|
font-size: 1.1em;
|
|
font-weight: 600;
|
|
color: #333;
|
|
}
|
|
|
|
.instance-grid {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
|
gap: 15px;
|
|
}
|
|
|
|
.instance-metric {
|
|
display: flex;
|
|
flex-direction: column;
|
|
}
|
|
|
|
.instance-metric-label {
|
|
font-size: 0.8em;
|
|
color: #666;
|
|
margin-bottom: 4px;
|
|
}
|
|
|
|
.instance-metric-value {
|
|
font-size: 1.1em;
|
|
font-weight: 600;
|
|
color: #333;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<header>
|
|
<div class="container">
|
|
<div style="display: flex; justify-content: space-between; align-items: center; flex-wrap: wrap; gap: 15px;">
|
|
<div>
|
|
<h1>GraphQL Proxy Admin Dashboard</h1>
|
|
<div class="subtitle">
|
|
Real-time monitoring and management
|
|
<span class="ws-status ws-disconnected" id="ws-status">Connecting...</span>
|
|
</div>
|
|
</div>
|
|
<div>
|
|
<input type="checkbox" id="cluster-mode-toggle">
|
|
<label for="cluster-mode-toggle" class="cluster-toggle-container">
|
|
<div class="toggle-switch"></div>
|
|
<div>
|
|
<div class="cluster-toggle-label">Cluster View</div>
|
|
<div class="cluster-toggle-info" id="cluster-info">Checking availability...</div>
|
|
</div>
|
|
</label>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</header>
|
|
|
|
<div class="container">
|
|
<!-- Cluster Status (shown when cluster mode detected) -->
|
|
<div id="cluster-status-section" style="display: none;">
|
|
<h2 class="section-title">Cluster Status</h2>
|
|
<div class="stats-grid">
|
|
<div class="card">
|
|
<div class="card-title">Total Instances</div>
|
|
<div class="card-value" id="cluster-total-instances">--</div>
|
|
<div class="card-label">Proxy nodes</div>
|
|
</div>
|
|
<div class="card">
|
|
<div class="card-title">Healthy Instances</div>
|
|
<div class="card-value" id="cluster-healthy-instances">--</div>
|
|
<div class="card-label">Active nodes</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- System Overview -->
|
|
<h2 class="section-title">
|
|
<span id="overview-title">System Overview</span>
|
|
</h2>
|
|
<div class="stats-grid">
|
|
<div class="card">
|
|
<div class="card-title">Uptime</div>
|
|
<div class="card-value" id="uptime">--</div>
|
|
<div class="card-label" id="uptime-seconds">-- seconds</div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<div class="card-title">Total Requests</div>
|
|
<div class="card-value" id="total-requests">--</div>
|
|
<div class="card-label">
|
|
<span style="color: #10b981;">✓ <span id="succeeded-requests">--</span></span>
|
|
<span style="color: #ef4444;">✗ <span id="failed-requests">--</span></span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<div class="card-title">Current RPS</div>
|
|
<div class="card-value" id="current-rps">--</div>
|
|
<div class="card-label">Avg: <span id="avg-rps">--</span> req/s</div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<div class="card-title">Success Rate</div>
|
|
<div class="card-value" id="success-rate">--%</div>
|
|
<div class="progress-bar">
|
|
<div class="progress-fill" id="success-progress" style="width: 0%"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Cache Statistics -->
|
|
<h2 class="section-title">Cache Performance</h2>
|
|
<div class="stats-grid">
|
|
<div class="card">
|
|
<div class="card-title">Cache Hit Rate</div>
|
|
<div class="card-value" id="cache-hit-rate">--%</div>
|
|
<div class="progress-bar">
|
|
<div class="progress-fill" id="cache-hit-progress" style="width: 0%"></div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<div class="card-title">Cache Hits / Misses</div>
|
|
<div class="card-value" id="cache-hits">--</div>
|
|
<div class="card-label">
|
|
Hits: <span id="cache-hits-detail">--</span> |
|
|
Misses: <span id="cache-misses">--</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<div class="card-title">Cached Queries</div>
|
|
<div class="card-value" id="cached-queries">--</div>
|
|
<div class="card-label">Total entries</div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<div class="card-title">Memory Usage</div>
|
|
<div class="card-value" id="cache-memory">-- MB</div>
|
|
<div class="card-label" id="cache-memory-pct">--%</div>
|
|
<div class="progress-bar">
|
|
<div class="progress-fill" id="memory-progress" style="width: 0%"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Real-time Charts -->
|
|
<h2 class="section-title">Real-time Metrics</h2>
|
|
<div class="stats-grid">
|
|
<div class="card">
|
|
<div class="card-title">Requests Per Second</div>
|
|
<div class="chart-container">
|
|
<canvas id="rps-chart"></canvas>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<div class="card-title">Cache Hit Rate Over Time</div>
|
|
<div class="chart-container">
|
|
<canvas id="cache-chart"></canvas>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Health Status -->
|
|
<h2 class="section-title">Health Status</h2>
|
|
<div class="card" id="health-card">
|
|
<div class="metric-row">
|
|
<span class="metric-label">Backend Status</span>
|
|
<span>
|
|
<span class="status-indicator status-unknown loading" id="health-indicator"></span>
|
|
<span id="health-status">Loading...</span>
|
|
</span>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Key Metrics -->
|
|
<h2 class="section-title">Advanced Features</h2>
|
|
<div class="stats-grid">
|
|
<div class="card">
|
|
<div class="card-title">Request Coalescing</div>
|
|
<div class="card-value" id="coalescing-rate">--%</div>
|
|
<div class="card-label">Backend Savings</div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<div class="card-title">Retry Budget</div>
|
|
<div class="card-value" id="retry-tokens">--</div>
|
|
<div class="card-label">Available Tokens</div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<div class="card-title">WebSocket Connections</div>
|
|
<div class="card-value" id="ws-connections">--</div>
|
|
<div class="card-label">Active Connections</div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<div class="card-title">Connection Pool</div>
|
|
<div class="card-value" id="pool-connections">--</div>
|
|
<div class="card-label">Active Connections</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Circuit Breaker -->
|
|
<h2 class="section-title">Circuit Breaker</h2>
|
|
<div class="card" id="circuit-breaker-card">
|
|
<div class="metric-row">
|
|
<span class="metric-label">Status</span>
|
|
<span class="metric-value" id="cb-state">
|
|
<span class="badge badge-info loading">Loading...</span>
|
|
</span>
|
|
</div>
|
|
<div class="metric-row">
|
|
<span class="metric-label">Enabled</span>
|
|
<span class="metric-value" id="cb-enabled">--</span>
|
|
</div>
|
|
<div class="metric-row">
|
|
<span class="metric-label">Max Failures</span>
|
|
<span class="metric-value" id="cb-max-failures">--</span>
|
|
</div>
|
|
<div class="metric-row">
|
|
<span class="metric-label">Timeout</span>
|
|
<span class="metric-value" id="cb-timeout">--s</span>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Request Coalescing Details -->
|
|
<h2 class="section-title">Request Coalescing</h2>
|
|
<div class="card">
|
|
<div class="metric-row">
|
|
<span class="metric-label">Total Requests</span>
|
|
<span class="metric-value" id="coalescing-total">--</span>
|
|
</div>
|
|
<div class="metric-row">
|
|
<span class="metric-label">Primary Requests</span>
|
|
<span class="metric-value" id="coalescing-primary">--</span>
|
|
</div>
|
|
<div class="metric-row">
|
|
<span class="metric-label">Coalesced Requests</span>
|
|
<span class="metric-value" id="coalescing-coalesced">--</span>
|
|
</div>
|
|
<div class="metric-row">
|
|
<span class="metric-label">Backend Savings</span>
|
|
<span class="metric-value" id="coalescing-savings">--%</span>
|
|
</div>
|
|
<div style="margin-top: 20px;">
|
|
<button class="btn" onclick="resetCoalescing()">Reset Statistics</button>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Retry Budget Details -->
|
|
<h2 class="section-title">Retry Budget</h2>
|
|
<div class="card">
|
|
<div class="metric-row">
|
|
<span class="metric-label">Current Tokens</span>
|
|
<span class="metric-value" id="retry-current-tokens">--</span>
|
|
</div>
|
|
<div class="metric-row">
|
|
<span class="metric-label">Max Tokens</span>
|
|
<span class="metric-value" id="retry-max-tokens">--</span>
|
|
</div>
|
|
<div class="metric-row">
|
|
<span class="metric-label">Total Attempts</span>
|
|
<span class="metric-value" id="retry-total">--</span>
|
|
</div>
|
|
<div class="metric-row">
|
|
<span class="metric-label">Denied Retries</span>
|
|
<span class="metric-value" id="retry-denied">--</span>
|
|
</div>
|
|
<div class="metric-row">
|
|
<span class="metric-label">Denial Rate</span>
|
|
<span class="metric-value" id="retry-denial-rate">--%</span>
|
|
</div>
|
|
<div style="margin-top: 20px;">
|
|
<button class="btn" onclick="resetRetryBudget()">Reset Statistics</button>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Instance Details (shown in cluster mode) -->
|
|
<div id="instance-details-section" style="display: none;">
|
|
<h2 class="section-title">Instance Details</h2>
|
|
<div id="instance-list"></div>
|
|
</div>
|
|
|
|
<div class="refresh-info" id="refresh-info">
|
|
<span id="connection-mode">Connecting to real-time updates...</span>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
// Chart instances
|
|
let rpsChart, cacheChart;
|
|
let rpsData = { labels: [], data: [] };
|
|
let cacheData = { labels: [], data: [] };
|
|
const MAX_DATA_POINTS = 60; // Keep last 60 data points
|
|
|
|
// WebSocket connection
|
|
let ws = null;
|
|
let wsReconnectInterval = null;
|
|
let useWebSocket = true;
|
|
|
|
// Cluster mode state
|
|
let clusterModeEnabled = false;
|
|
let clusterModeAvailable = false;
|
|
|
|
// Smoothing buffers for metrics (10-second moving average at 2s intervals = 5 data points)
|
|
const smoothingWindow = 5;
|
|
const rpsBuffer = [];
|
|
const successRateBuffer = [];
|
|
const cacheHitRateBuffer = [];
|
|
|
|
// Initialize charts
|
|
function initCharts() {
|
|
const rpsCtx = document.getElementById('rps-chart').getContext('2d');
|
|
rpsChart = new Chart(rpsCtx, {
|
|
type: 'line',
|
|
data: {
|
|
labels: [],
|
|
datasets: [{
|
|
label: 'Requests/sec',
|
|
data: [],
|
|
borderColor: '#667eea',
|
|
backgroundColor: 'rgba(102, 126, 234, 0.1)',
|
|
tension: 0.4,
|
|
fill: true
|
|
}]
|
|
},
|
|
options: {
|
|
responsive: true,
|
|
maintainAspectRatio: false,
|
|
plugins: {
|
|
legend: { display: false }
|
|
},
|
|
scales: {
|
|
y: {
|
|
beginAtZero: true,
|
|
ticks: { precision: 0 }
|
|
},
|
|
x: {
|
|
display: false
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
const cacheCtx = document.getElementById('cache-chart').getContext('2d');
|
|
cacheChart = new Chart(cacheCtx, {
|
|
type: 'line',
|
|
data: {
|
|
labels: [],
|
|
datasets: [{
|
|
label: 'Hit Rate %',
|
|
data: [],
|
|
borderColor: '#10b981',
|
|
backgroundColor: 'rgba(16, 185, 129, 0.1)',
|
|
tension: 0.4,
|
|
fill: true
|
|
}]
|
|
},
|
|
options: {
|
|
responsive: true,
|
|
maintainAspectRatio: false,
|
|
plugins: {
|
|
legend: { display: false }
|
|
},
|
|
scales: {
|
|
y: {
|
|
beginAtZero: true,
|
|
max: 100,
|
|
ticks: {
|
|
callback: function(value) {
|
|
return value + '%';
|
|
}
|
|
}
|
|
},
|
|
x: {
|
|
display: false
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
// Update chart data
|
|
function updateChart(chart, dataStore, value) {
|
|
const timestamp = new Date().toLocaleTimeString();
|
|
|
|
dataStore.labels.push(timestamp);
|
|
dataStore.data.push(value);
|
|
|
|
// Keep only last MAX_DATA_POINTS
|
|
if (dataStore.labels.length > MAX_DATA_POINTS) {
|
|
dataStore.labels.shift();
|
|
dataStore.data.shift();
|
|
}
|
|
|
|
chart.data.labels = dataStore.labels;
|
|
chart.data.datasets[0].data = dataStore.data;
|
|
chart.update('none'); // Update without animation for smoother real-time updates
|
|
}
|
|
|
|
// Connect to WebSocket
|
|
function connectWebSocket() {
|
|
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
|
|
const wsUrl = `${protocol}//${window.location.host}/admin/ws/stats`;
|
|
|
|
try {
|
|
ws = new WebSocket(wsUrl);
|
|
|
|
ws.onopen = () => {
|
|
updateWSStatus(true);
|
|
if (wsReconnectInterval) {
|
|
clearInterval(wsReconnectInterval);
|
|
wsReconnectInterval = null;
|
|
}
|
|
};
|
|
|
|
ws.onmessage = (event) => {
|
|
try {
|
|
const data = JSON.parse(event.data);
|
|
updateAllStats(data);
|
|
} catch (error) {
|
|
console.error('Failed to parse WebSocket message:', error);
|
|
}
|
|
};
|
|
|
|
ws.onerror = (error) => {
|
|
console.error('WebSocket error:', error);
|
|
};
|
|
|
|
ws.onclose = () => {
|
|
updateWSStatus(false);
|
|
// Try to reconnect after 5 seconds
|
|
if (!wsReconnectInterval) {
|
|
wsReconnectInterval = setInterval(() => {
|
|
connectWebSocket();
|
|
}, 5000);
|
|
}
|
|
};
|
|
} catch (error) {
|
|
console.error('Failed to create WebSocket:', error);
|
|
updateWSStatus(false);
|
|
// Fall back to polling if WebSocket fails
|
|
useWebSocket = false;
|
|
startPolling();
|
|
}
|
|
}
|
|
|
|
// Update WebSocket status indicator
|
|
function updateWSStatus(connected) {
|
|
const statusEl = document.getElementById('ws-status');
|
|
const infoEl = document.getElementById('connection-mode');
|
|
|
|
if (connected) {
|
|
statusEl.className = 'ws-status ws-connected';
|
|
statusEl.textContent = 'Live';
|
|
infoEl.textContent = 'Real-time updates via WebSocket';
|
|
} else {
|
|
statusEl.className = 'ws-status ws-disconnected';
|
|
statusEl.textContent = 'Reconnecting...';
|
|
infoEl.textContent = 'Attempting to reconnect...';
|
|
}
|
|
}
|
|
|
|
// Fallback: Fetch and update dashboard data via polling
|
|
async function updateDashboard() {
|
|
try {
|
|
if (clusterModeEnabled) {
|
|
// Fetch cluster stats
|
|
const [clusterStats, instances] = await Promise.all([
|
|
fetch('/admin/api/cluster/stats').then(r => r.json()),
|
|
fetch('/admin/api/cluster/instances').then(r => r.json())
|
|
]);
|
|
|
|
if (clusterStats.cluster_mode) {
|
|
updateClusterStats(clusterStats, instances);
|
|
}
|
|
} else {
|
|
// Fetch all stats for single instance
|
|
const [stats, health, cb, cache, coalescing, retryBudget, wsStats, connections] = await Promise.all([
|
|
fetch('/admin/api/stats').then(r => r.json()),
|
|
fetch('/admin/api/health').then(r => r.json()),
|
|
fetch('/admin/api/circuit-breaker').then(r => r.json()),
|
|
fetch('/admin/api/cache').then(r => r.json()),
|
|
fetch('/admin/api/coalescing').then(r => r.json()),
|
|
fetch('/admin/api/retry-budget').then(r => r.json()),
|
|
fetch('/admin/api/websocket').then(r => r.json()),
|
|
fetch('/admin/api/connections').then(r => r.json())
|
|
]);
|
|
|
|
const allData = {
|
|
stats,
|
|
health,
|
|
circuit_breaker: cb,
|
|
cache,
|
|
coalescing,
|
|
retry_budget: retryBudget,
|
|
websocket: wsStats,
|
|
connections
|
|
};
|
|
|
|
updateAllStats(allData);
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('Failed to update dashboard:', error);
|
|
}
|
|
}
|
|
|
|
// Check if cluster mode is available
|
|
async function checkClusterMode() {
|
|
try {
|
|
const response = await fetch('/admin/api/cluster/stats');
|
|
const data = await response.json();
|
|
|
|
if (response.ok && data.cluster_mode) {
|
|
clusterModeAvailable = true;
|
|
document.getElementById('cluster-info').textContent =
|
|
`(${data.total_instances} instance${data.total_instances !== 1 ? 's' : ''} available)`;
|
|
return true;
|
|
}
|
|
} catch (error) {
|
|
// Cluster mode not available
|
|
}
|
|
|
|
clusterModeAvailable = false;
|
|
document.getElementById('cluster-info').textContent = '(not available)';
|
|
document.getElementById('cluster-mode-toggle').disabled = true;
|
|
return false;
|
|
}
|
|
|
|
// Update cluster statistics
|
|
function updateClusterStats(clusterData, instancesData) {
|
|
// Show cluster sections
|
|
document.getElementById('cluster-status-section').style.display = 'block';
|
|
document.getElementById('instance-details-section').style.display = 'block';
|
|
document.getElementById('overview-title').textContent = 'Cluster Overview';
|
|
|
|
// Update cluster status
|
|
document.getElementById('cluster-total-instances').textContent = clusterData.total_instances || 0;
|
|
document.getElementById('cluster-healthy-instances').textContent = clusterData.healthy_instances || 0;
|
|
|
|
// Update cluster info in toggle label
|
|
const totalInstances = clusterData.total_instances || 0;
|
|
document.getElementById('cluster-info').textContent =
|
|
`(${totalInstances} instance${totalInstances !== 1 ? 's' : ''} available)`;
|
|
|
|
// Update combined stats
|
|
if (clusterData.stats) {
|
|
updateStats({ ...clusterData.stats, cluster_mode: true });
|
|
|
|
// Update memory usage from cluster stats
|
|
if (clusterData.stats.memory) {
|
|
updateCacheStats({
|
|
memory_usage_mb: clusterData.stats.memory.total_usage_mb || 0
|
|
});
|
|
}
|
|
}
|
|
|
|
// Update instance list
|
|
if (instancesData && instancesData.instances) {
|
|
updateInstanceList(instancesData.instances, instancesData.current_instance);
|
|
}
|
|
}
|
|
|
|
// Update individual instance list
|
|
function updateInstanceList(instances, currentInstanceID) {
|
|
const container = document.getElementById('instance-list');
|
|
container.innerHTML = '';
|
|
|
|
instances.forEach(instance => {
|
|
// Use JSON field names (snake_case), not Go struct names (PascalCase)
|
|
const isCurrent = instance.instance_id === currentInstanceID;
|
|
const isHealthy = instance.health && instance.health.status === 'healthy';
|
|
|
|
const card = document.createElement('div');
|
|
card.className = 'instance-card';
|
|
|
|
const header = document.createElement('div');
|
|
header.className = 'instance-header';
|
|
|
|
const title = document.createElement('div');
|
|
title.className = 'instance-title';
|
|
title.innerHTML = `
|
|
<span class="status-indicator ${isHealthy ? 'status-healthy' : 'status-unhealthy'}"></span>
|
|
${instance.hostname || 'unknown'}
|
|
${isCurrent ? '<span class="badge badge-info" style="margin-left: 8px;">Current</span>' : ''}
|
|
`;
|
|
|
|
const uptime = document.createElement('span');
|
|
uptime.style.fontSize = '0.85em';
|
|
uptime.style.color = '#666';
|
|
uptime.textContent = `Uptime: ${formatUptime(instance.uptime_seconds || 0)}`;
|
|
|
|
header.appendChild(title);
|
|
header.appendChild(uptime);
|
|
|
|
const grid = document.createElement('div');
|
|
grid.className = 'instance-grid';
|
|
|
|
// Extract stats - use JSON field names
|
|
const stats = instance.stats || {};
|
|
const requests = stats.requests || {};
|
|
const cache = instance.cache_summary || stats.cache_summary || {};
|
|
|
|
const failedCount = requests.failed || 0;
|
|
const totalCount = requests.total || 0;
|
|
const failureInfo = failedCount > 0 ? ` (${failedCount} failed)` : '';
|
|
|
|
const metrics = [
|
|
{
|
|
label: 'Total Requests',
|
|
value: formatNumber(totalCount),
|
|
title: totalCount.toLocaleString() + ' total requests' + failureInfo
|
|
},
|
|
{
|
|
label: 'Success Rate',
|
|
value: (requests.success_rate_pct || 0).toFixed(1) + '%',
|
|
title: null
|
|
},
|
|
{
|
|
label: 'Current RPS',
|
|
value: (requests.current_requests_per_second || 0).toFixed(1),
|
|
title: null
|
|
},
|
|
{
|
|
label: 'Cache Hit Rate',
|
|
value: (cache.hit_rate_pct || 0).toFixed(1) + '%',
|
|
title: null
|
|
}
|
|
];
|
|
|
|
metrics.forEach(metric => {
|
|
const metricDiv = document.createElement('div');
|
|
metricDiv.className = 'instance-metric';
|
|
if (metric.title) {
|
|
metricDiv.title = metric.title;
|
|
}
|
|
metricDiv.innerHTML = `
|
|
<div class="instance-metric-label">${metric.label}</div>
|
|
<div class="instance-metric-value">${metric.value}</div>
|
|
`;
|
|
grid.appendChild(metricDiv);
|
|
});
|
|
|
|
card.appendChild(header);
|
|
card.appendChild(grid);
|
|
container.appendChild(card);
|
|
});
|
|
}
|
|
|
|
// Format uptime in human readable format
|
|
function formatUptime(seconds) {
|
|
const days = Math.floor(seconds / 86400);
|
|
const hours = Math.floor((seconds % 86400) / 3600);
|
|
const minutes = Math.floor((seconds % 3600) / 60);
|
|
|
|
if (days > 0) return `${days}d ${hours}h`;
|
|
if (hours > 0) return `${hours}h ${minutes}m`;
|
|
return `${minutes}m`;
|
|
}
|
|
|
|
// Format large numbers compactly (1.2M, 3.4K, etc)
|
|
function formatNumber(num) {
|
|
if (num === undefined || num === null) return '0';
|
|
|
|
const absNum = Math.abs(num);
|
|
if (absNum >= 1000000000) {
|
|
return (num / 1000000000).toFixed(1) + 'B';
|
|
}
|
|
if (absNum >= 1000000) {
|
|
return (num / 1000000).toFixed(1) + 'M';
|
|
}
|
|
if (absNum >= 1000) {
|
|
return (num / 1000).toFixed(1) + 'K';
|
|
}
|
|
return num.toLocaleString();
|
|
}
|
|
|
|
// Smooth metric values using moving average
|
|
function smoothMetric(buffer, newValue) {
|
|
buffer.push(newValue);
|
|
if (buffer.length > smoothingWindow) {
|
|
buffer.shift(); // Remove oldest value
|
|
}
|
|
// Calculate average
|
|
const sum = buffer.reduce((a, b) => a + b, 0);
|
|
return sum / buffer.length;
|
|
}
|
|
|
|
// Get trend indicator (↑ ↗ → ↘ ↓)
|
|
function getTrendIndicator(buffer) {
|
|
if (buffer.length < 2) return '→';
|
|
const recent = buffer.slice(-3); // Last 3 values
|
|
const avg = recent.reduce((a, b) => a + b, 0) / recent.length;
|
|
const diff = recent[recent.length - 1] - recent[0];
|
|
const percentChange = Math.abs(diff / (avg || 1)) * 100;
|
|
|
|
if (percentChange < 5) return '→'; // Stable
|
|
if (diff > 0) {
|
|
return percentChange > 15 ? '↑' : '↗'; // Strong/moderate increase
|
|
} else {
|
|
return percentChange > 15 ? '↓' : '↘'; // Strong/moderate decrease
|
|
}
|
|
}
|
|
|
|
// Update all statistics
|
|
function updateAllStats(data) {
|
|
if (data.stats) updateStats(data.stats);
|
|
if (data.health) updateHealth(data.health);
|
|
if (data.circuit_breaker) updateCircuitBreaker(data.circuit_breaker);
|
|
if (data.cache) updateCacheStats(data.cache);
|
|
if (data.coalescing) updateCoalescing(data.coalescing);
|
|
if (data.retry_budget) updateRetryBudget(data.retry_budget);
|
|
if (data.websocket) updateWebSocket(data.websocket);
|
|
if (data.connections) updateConnections(data.connections);
|
|
}
|
|
|
|
// Update main stats
|
|
function updateStats(data) {
|
|
// Uptime
|
|
document.getElementById('uptime').textContent = data.uptime_human || '--';
|
|
document.getElementById('uptime-seconds').textContent =
|
|
(data.uptime_seconds || 0).toFixed(0) + ' seconds';
|
|
|
|
if (data.requests) {
|
|
const req = data.requests;
|
|
|
|
// Total requests with compact formatting
|
|
document.getElementById('total-requests').textContent = formatNumber(req.total || 0);
|
|
document.getElementById('total-requests').title = (req.total || 0).toLocaleString() + ' total requests';
|
|
|
|
document.getElementById('succeeded-requests').textContent = formatNumber(req.succeeded || 0);
|
|
document.getElementById('succeeded-requests').title = (req.succeeded || 0).toLocaleString() + ' succeeded';
|
|
|
|
document.getElementById('failed-requests').textContent = formatNumber(req.failed || 0);
|
|
document.getElementById('failed-requests').title = (req.failed || 0).toLocaleString() + ' failed';
|
|
|
|
// Show failure details if there are failures
|
|
if (req.failed > 0) {
|
|
const failureRate = (req.failed / (req.total || 1) * 100).toFixed(2);
|
|
document.getElementById('failed-requests').title += ` (${failureRate}% failure rate)`;
|
|
}
|
|
|
|
// Success rate with smoothing
|
|
const rawSuccessRate = req.success_rate_pct || 0;
|
|
const smoothedSuccessRate = smoothMetric(successRateBuffer, rawSuccessRate);
|
|
const successTrend = getTrendIndicator(successRateBuffer);
|
|
|
|
document.getElementById('success-rate').textContent =
|
|
smoothedSuccessRate.toFixed(1) + '% ' + successTrend;
|
|
document.getElementById('success-rate').title =
|
|
`10s avg: ${smoothedSuccessRate.toFixed(2)}% | Current: ${rawSuccessRate.toFixed(2)}%`;
|
|
document.getElementById('success-progress').style.width =
|
|
smoothedSuccessRate + '%';
|
|
|
|
// RPS with smoothing
|
|
const rawRPS = req.current_requests_per_second || 0;
|
|
const smoothedRPS = smoothMetric(rpsBuffer, rawRPS);
|
|
const rpsTrend = getTrendIndicator(rpsBuffer);
|
|
|
|
document.getElementById('current-rps').textContent =
|
|
smoothedRPS.toFixed(1) + ' ' + rpsTrend;
|
|
document.getElementById('current-rps').title =
|
|
`10s avg: ${smoothedRPS.toFixed(2)} | Current: ${rawRPS.toFixed(2)}`;
|
|
|
|
document.getElementById('avg-rps').textContent =
|
|
(req.avg_requests_per_second || 0).toFixed(1);
|
|
|
|
// Update RPS chart with smoothed value
|
|
updateChart(rpsChart, rpsData, smoothedRPS);
|
|
}
|
|
|
|
// Cache summary with smoothing
|
|
if (data.cache_summary) {
|
|
const cache = data.cache_summary;
|
|
|
|
const rawHitRate = cache.hit_rate_pct || 0;
|
|
const smoothedHitRate = smoothMetric(cacheHitRateBuffer, rawHitRate);
|
|
const hitRateTrend = getTrendIndicator(cacheHitRateBuffer);
|
|
|
|
document.getElementById('cache-hit-rate').textContent =
|
|
smoothedHitRate.toFixed(1) + '% ' + hitRateTrend;
|
|
document.getElementById('cache-hit-rate').title =
|
|
`10s avg: ${smoothedHitRate.toFixed(2)}% | Current: ${rawHitRate.toFixed(2)}%`;
|
|
document.getElementById('cache-hit-progress').style.width =
|
|
smoothedHitRate + '%';
|
|
|
|
document.getElementById('cache-hits').textContent = formatNumber(cache.hits || 0);
|
|
document.getElementById('cache-hits').title = (cache.hits || 0).toLocaleString() + ' cache hits';
|
|
|
|
document.getElementById('cache-hits-detail').textContent = formatNumber(cache.hits || 0);
|
|
document.getElementById('cache-hits-detail').title = (cache.hits || 0).toLocaleString();
|
|
|
|
document.getElementById('cache-misses').textContent = formatNumber(cache.misses || 0);
|
|
document.getElementById('cache-misses').title = (cache.misses || 0).toLocaleString() + ' cache misses';
|
|
|
|
document.getElementById('cached-queries').textContent = formatNumber(cache.total_cached || 0);
|
|
document.getElementById('cached-queries').title = (cache.total_cached || 0).toLocaleString() + ' unique queries cached';
|
|
|
|
// Update cache hit rate chart with smoothed value
|
|
updateChart(cacheChart, cacheData, smoothedHitRate);
|
|
}
|
|
}
|
|
|
|
// Update cache detailed stats
|
|
function updateCacheStats(data) {
|
|
if (data.memory_usage_mb !== undefined) {
|
|
if (data.memory_usage_mb === -1) {
|
|
// Redis cache - memory tracking not available
|
|
document.getElementById('cache-memory').textContent = 'N/A';
|
|
document.getElementById('cache-memory').title = 'Memory tracking not available for Redis cache';
|
|
document.getElementById('cache-memory-pct').textContent = 'Redis cache';
|
|
document.getElementById('memory-progress').style.width = '0%';
|
|
} else {
|
|
document.getElementById('cache-memory').textContent =
|
|
data.memory_usage_mb.toFixed(2) + ' MB';
|
|
document.getElementById('cache-memory').title = 'In-memory cache usage';
|
|
|
|
if (data.memory_usage_pct !== undefined) {
|
|
const memPct = data.memory_usage_pct;
|
|
document.getElementById('cache-memory-pct').textContent =
|
|
memPct.toFixed(1) + '% used';
|
|
document.getElementById('memory-progress').style.width =
|
|
memPct + '%';
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function updateHealth(data) {
|
|
const indicator = document.getElementById('health-indicator');
|
|
const status = document.getElementById('health-status');
|
|
|
|
indicator.classList.remove('loading');
|
|
|
|
if (data.status === 'healthy') {
|
|
indicator.className = 'status-indicator status-healthy';
|
|
status.textContent = 'System Healthy';
|
|
} else if (data.status === 'unhealthy') {
|
|
indicator.className = 'status-indicator status-unhealthy';
|
|
status.textContent = 'System Unhealthy';
|
|
} else {
|
|
indicator.className = 'status-indicator status-unknown';
|
|
status.textContent = 'Status Unknown';
|
|
}
|
|
}
|
|
|
|
function updateCircuitBreaker(data) {
|
|
const stateEl = document.getElementById('cb-state');
|
|
stateEl.classList.remove('loading');
|
|
|
|
let badgeClass = 'badge-info';
|
|
if (data.state === 'closed') badgeClass = 'badge-success';
|
|
else if (data.state === 'open') badgeClass = 'badge-danger';
|
|
else if (data.state === 'half-open') badgeClass = 'badge-warning';
|
|
|
|
stateEl.innerHTML = `<span class="badge ${badgeClass}">${data.state || 'Unknown'}</span>`;
|
|
|
|
document.getElementById('cb-enabled').textContent = data.enabled ? 'Yes' : 'No';
|
|
|
|
if (data.config) {
|
|
document.getElementById('cb-max-failures').textContent = data.config.max_failures || '--';
|
|
document.getElementById('cb-timeout').textContent = (data.config.timeout || '--') + 's';
|
|
}
|
|
}
|
|
|
|
function updateCoalescing(data) {
|
|
document.getElementById('coalescing-rate').textContent =
|
|
(data.backend_savings_pct || 0).toFixed(1) + '%';
|
|
document.getElementById('coalescing-total').textContent =
|
|
(data.total_requests || 0).toLocaleString();
|
|
document.getElementById('coalescing-primary').textContent =
|
|
(data.primary_requests || 0).toLocaleString();
|
|
document.getElementById('coalescing-coalesced').textContent =
|
|
(data.coalesced_requests || 0).toLocaleString();
|
|
document.getElementById('coalescing-savings').textContent =
|
|
(data.backend_savings_pct || 0).toFixed(1) + '%';
|
|
}
|
|
|
|
function updateRetryBudget(data) {
|
|
document.getElementById('retry-tokens').textContent =
|
|
data.current_tokens || '--';
|
|
document.getElementById('retry-current-tokens').textContent =
|
|
data.current_tokens || '--';
|
|
document.getElementById('retry-max-tokens').textContent =
|
|
data.max_tokens || '--';
|
|
document.getElementById('retry-total').textContent =
|
|
(data.total_attempts || 0).toLocaleString();
|
|
document.getElementById('retry-denied').textContent =
|
|
(data.denied_retries || 0).toLocaleString();
|
|
document.getElementById('retry-denial-rate').textContent =
|
|
(data.denial_rate_pct || 0).toFixed(2) + '%';
|
|
}
|
|
|
|
function updateWebSocket(data) {
|
|
document.getElementById('ws-connections').textContent =
|
|
data.active_connections || 0;
|
|
}
|
|
|
|
function updateConnections(data) {
|
|
document.getElementById('pool-connections').textContent =
|
|
data.active_connections || 0;
|
|
}
|
|
|
|
async function resetCoalescing() {
|
|
try {
|
|
await fetch('/admin/api/coalescing/reset', { method: 'POST' });
|
|
updateDashboard();
|
|
} catch (error) {
|
|
alert('Failed to reset coalescing statistics');
|
|
}
|
|
}
|
|
|
|
async function resetRetryBudget() {
|
|
try {
|
|
await fetch('/admin/api/retry-budget/reset', { method: 'POST' });
|
|
updateDashboard();
|
|
} catch (error) {
|
|
alert('Failed to reset retry budget statistics');
|
|
}
|
|
}
|
|
|
|
// Start polling (fallback when WebSocket is not available)
|
|
function startPolling() {
|
|
const statusEl = document.getElementById('ws-status');
|
|
const infoEl = document.getElementById('connection-mode');
|
|
|
|
statusEl.className = 'ws-status ws-disconnected';
|
|
statusEl.textContent = 'Polling';
|
|
infoEl.textContent = 'Updates every 5 seconds (WebSocket unavailable)';
|
|
|
|
// Initial load
|
|
updateDashboard();
|
|
|
|
// Refresh every 5 seconds
|
|
setInterval(updateDashboard, 5000);
|
|
}
|
|
|
|
// Initialize dashboard
|
|
async function initDashboard() {
|
|
// Initialize charts first
|
|
initCharts();
|
|
|
|
// Check if cluster mode is available
|
|
await checkClusterMode();
|
|
|
|
// Setup cluster mode toggle
|
|
const toggle = document.getElementById('cluster-mode-toggle');
|
|
toggle.addEventListener('change', (e) => {
|
|
clusterModeEnabled = e.target.checked && clusterModeAvailable;
|
|
|
|
// Toggle cluster sections visibility
|
|
if (clusterModeEnabled) {
|
|
document.getElementById('cluster-status-section').style.display = 'block';
|
|
document.getElementById('overview-title').textContent = 'Cluster Overview';
|
|
} else {
|
|
document.getElementById('cluster-status-section').style.display = 'none';
|
|
document.getElementById('instance-details-section').style.display = 'none';
|
|
document.getElementById('overview-title').textContent = 'System Overview';
|
|
}
|
|
|
|
// Refresh data
|
|
updateDashboard();
|
|
});
|
|
|
|
// Try WebSocket connection first
|
|
connectWebSocket();
|
|
|
|
// Set a timeout to fall back to polling if WebSocket doesn't connect
|
|
setTimeout(() => {
|
|
if (!ws || ws.readyState !== WebSocket.OPEN) {
|
|
useWebSocket = false;
|
|
startPolling();
|
|
}
|
|
}, 3000);
|
|
}
|
|
|
|
// Start when page loads
|
|
if (document.readyState === 'loading') {
|
|
document.addEventListener('DOMContentLoaded', initDashboard);
|
|
} else {
|
|
initDashboard();
|
|
}
|
|
</script>
|
|
</body>
|
|
</html> |