← back to Exo
nice dashboard, add benchmark results tokens per second
6f8582d8251c24cb5be6455793da06e93c4e1d59 · 2024-11-25 17:30:45 +0400 · Alex Cheema
Files touched
M extra/dashboard/dashboard.py
Diff
commit 6f8582d8251c24cb5be6455793da06e93c4e1d59
Author: Alex Cheema <alexcheema123@gmail.com>
Date: Mon Nov 25 17:30:45 2024 +0400
nice dashboard, add benchmark results tokens per second
---
extra/dashboard/dashboard.py | 576 ++++++++++++++++++++++++++++++++++++++++---
1 file changed, 544 insertions(+), 32 deletions(-)
diff --git a/extra/dashboard/dashboard.py b/extra/dashboard/dashboard.py
index e8101754..6150ad60 100644
--- a/extra/dashboard/dashboard.py
+++ b/extra/dashboard/dashboard.py
@@ -9,6 +9,7 @@ from typing import List, Dict, Optional
from pathlib import Path
from plotly.subplots import make_subplots
import plotly.graph_objects as go
+from datetime import datetime, timedelta
class AsyncCircleCIClient:
def __init__(self, token: str, project_slug: str):
@@ -26,19 +27,54 @@ class AsyncCircleCIClient:
response.raise_for_status()
return await response.json()
- async def get_recent_pipelines(self, session: aiohttp.ClientSession, limit: int = 100) -> List[Dict]:
- self.logger.info(f"Fetching {limit} recent pipelines...")
- url = f"{self.base_url}/project/{self.project_slug}/pipeline"
- params = {"limit": limit * 2}
+ async def get_recent_pipelines(
+ self,
+ session: aiohttp.ClientSession,
+ org_slug: str = None,
+ page_token: str = None,
+ limit: int = None,
+ branch: str = None
+ ):
+ """
+ Get recent pipelines for a project with pagination support
+
+ Args:
+ session: aiohttp client session
+ org_slug: Organization slug
+ page_token: Token for pagination
+ limit: Maximum number of pipelines to return
+ branch: Specific branch to fetch pipelines from
+ """
+
+ params = {
+ "branch": branch,
+ "page-token": page_token
+ }
+
+ # Remove None values
+ params = {k: v for k, v in params.items() if v is not None}
+ url = f"{self.base_url}/project/{self.project_slug}/pipeline"
data = await self.get_json(session, url, params)
- pipelines = [
- p for p in data["items"]
- if p["state"] == "created"
- and p.get("trigger_parameters", {}).get("git", {}).get("branch") == "main"
- ][:limit]
+ pipelines = data["items"]
+
+ next_page_token = data.get("next_page_token")
+
+ # If there are more pages and we haven't hit the limit, recursively get them
+ if next_page_token and (limit is None or len(pipelines) < limit):
+ next_pipelines = await self.get_recent_pipelines(
+ session,
+ org_slug,
+ page_token=next_page_token,
+ limit=limit,
+ branch=branch
+ )
+ pipelines.extend(next_pipelines)
+
+ # Trim to limit if needed
+ if limit is not None:
+ pipelines = pipelines[:limit]
- self.logger.info(f"Found {len(pipelines)} successful main branch pipelines")
return pipelines
async def get_workflow_jobs(self, session: aiohttp.ClientSession, pipeline_id: str) -> List[Dict]:
@@ -114,6 +150,12 @@ class PackageSizeTracker:
jobs = await self.client.get_workflow_jobs(session, pipeline["id"])
+ # Add test status check
+ test_job = next(
+ (j for j in jobs if j["name"] == "test" and j["status"] in ["success", "failed"]),
+ None
+ )
+
# Get package size data
size_job = next(
(j for j in jobs if j["name"] == "measure_pip_sizes" and j["status"] == "success"),
@@ -126,8 +168,14 @@ class PackageSizeTracker:
None
)
+ # Get benchmark data from runner job
+ benchmark_job = next(
+ (j for j in jobs if j["name"] == "runner" and j["status"] == "success"),
+ None
+ )
+
# Return None if no relevant jobs found
- if not size_job and not linecount_job:
+ if not size_job and not linecount_job and not benchmark_job:
self.logger.debug(f"No relevant jobs found for pipeline {pipeline['id']}")
return None
@@ -135,8 +183,28 @@ class PackageSizeTracker:
"commit_hash": commit_info['commit_hash'],
"commit_url": commit_info['web_url'],
"timestamp": pipeline.get("created_at", pipeline.get("updated_at")),
+ "tests_passing": test_job["status"] == "success" if test_job else None
}
+ # Process benchmark data if available
+ if benchmark_job:
+ benchmark_artifacts = await self.client.get_artifacts(session, benchmark_job["job_number"])
+ benchmark_report = next(
+ (a for a in benchmark_artifacts if a["path"].endswith("benchmark.json")),
+ None
+ )
+ if benchmark_report:
+ benchmark_data = await self.client.get_json(session, benchmark_report["url"])
+ data_point.update({
+ "tokens_per_second": benchmark_data["tokens_per_second"],
+ "time_to_first_token": benchmark_data["time_to_first_token"]
+ })
+ self.logger.info(
+ f"Processed benchmark data for pipeline {pipeline['id']}: "
+ f"commit {commit_info['commit_hash'][:7]}, "
+ f"tokens/s {benchmark_data['tokens_per_second']:.2f}"
+ )
+
# Process size data if available
if size_job:
size_artifacts = await self.client.get_artifacts(session, size_job["job_number"])
@@ -185,8 +253,27 @@ class PackageSizeTracker:
async def collect_data(self) -> List[Dict]:
self.logger.info("Starting data collection...")
async with aiohttp.ClientSession(headers=self.client.headers) as session:
- # Get pipelines
- pipelines = await self.client.get_recent_pipelines(session, 100)
+ # Get pipelines from both main and circleci branches
+ main_pipelines = await self.client.get_recent_pipelines(
+ session,
+ org_slug=self.client.project_slug,
+ limit=20,
+ branch="main"
+ )
+ circleci_pipelines = await self.client.get_recent_pipelines(
+ session,
+ org_slug=self.client.project_slug,
+ limit=20,
+ branch="circleci"
+ )
+
+ pipelines = main_pipelines + circleci_pipelines
+ # Sort pipelines by created_at date
+ pipelines.sort(key=lambda x: x.get("created_at", x.get("updated_at")), reverse=True)
+ # Take the 20 most recent pipelines after combining
+ pipelines = pipelines[:20]
+
+ self.logger.info(f"Found {len(pipelines)} recent pipelines")
# Process all pipelines in parallel
tasks = [self.process_pipeline(session, pipeline) for pipeline in pipelines]
@@ -206,6 +293,7 @@ class PackageSizeTracker:
# Create separate dataframes for each metric
df_size = pd.DataFrame([d for d in data if 'total_size_mb' in d])
df_lines = pd.DataFrame([d for d in data if 'total_lines' in d])
+ df_benchmark = pd.DataFrame([d for d in data if 'tokens_per_second' in d])
# Ensure output directory exists
output_dir = Path(output_dir)
@@ -213,11 +301,36 @@ class PackageSizeTracker:
# Create a single figure with subplots
fig = make_subplots(
- rows=2, cols=1,
- subplot_titles=('Package Size Trend', 'Line Count Trend'),
- vertical_spacing=0.2
+ rows=3, cols=2,
+ subplot_titles=('Test Status', 'Package Size Trend', '', 'Line Count Trend', '', 'Tokens per Second'),
+ vertical_spacing=0.2,
+ column_widths=[0.2, 0.8],
+ specs=[[{"type": "indicator"}, {"type": "scatter"}],
+ [None, {"type": "scatter"}],
+ [None, {"type": "scatter"}]]
)
+ # Add test status indicator if we have data
+ latest_test_status = next((d["tests_passing"] for d in reversed(data) if "tests_passing" in d), None)
+ if latest_test_status is not None:
+ fig.add_trace(
+ go.Indicator(
+ mode="gauge",
+ gauge={
+ "shape": "bullet",
+ "axis": {"visible": False},
+ "bar": {"color": "green" if latest_test_status else "red"},
+ "bgcolor": "white",
+ "steps": [
+ {"range": [0, 1], "color": "lightgray"}
+ ]
+ },
+ value=1,
+ title={"text": "Tests<br>Status"}
+ ),
+ row=1, col=1
+ )
+
# Add package size trace if we have data
if not df_size.empty:
df_size['timestamp'] = pd.to_datetime(df_size['timestamp'])
@@ -237,9 +350,9 @@ class PackageSizeTracker:
"<extra></extra>"
])
),
- row=1, col=1
+ row=1, col=2
)
- fig.update_yaxes(title_text="Size (MB)", row=1, col=1)
+ fig.update_yaxes(title_text="Size (MB)", row=1, col=2)
# Add line count trace if we have data
if not df_lines.empty:
@@ -260,40 +373,419 @@ class PackageSizeTracker:
"<extra></extra>"
])
),
- row=2, col=1
+ row=2, col=2
)
- fig.update_yaxes(title_text="Total Lines", row=2, col=1)
+ fig.update_yaxes(title_text="Total Lines", row=2, col=2)
+
+ # Add tokens per second trace if we have data
+ if not df_benchmark.empty:
+ df_benchmark['timestamp'] = pd.to_datetime(df_benchmark['timestamp'])
+ df_benchmark = df_benchmark.sort_values('timestamp')
+
+ fig.add_trace(
+ go.Scatter(
+ x=df_benchmark['timestamp'],
+ y=df_benchmark['tokens_per_second'],
+ mode='lines+markers',
+ name='Tokens/Second',
+ customdata=df_benchmark[['commit_hash', 'commit_url']].values,
+ hovertemplate="<br>".join([
+ "Tokens/s: %{y:.2f}",
+ "Date: %{x}",
+ "Commit: %{customdata[0]}",
+ "<extra></extra>"
+ ])
+ ),
+ row=3, col=2
+ )
+ fig.update_yaxes(title_text="Tokens per Second", row=3, col=2)
# Update layout
fig.update_layout(
- height=800, # Taller to accommodate both plots
+ height=800,
showlegend=False,
title_text="Package Metrics Dashboard",
title_x=0.5,
plot_bgcolor='white',
paper_bgcolor='white',
font=dict(size=12),
- hovermode='x unified',
- xaxis=dict(title_text="Date"),
- xaxis2=dict(title_text="Date")
+ hovermode='x unified'
)
- # Add click event handling
+ # Update the dashboard HTML with date range picker
dashboard_html = f"""
<html>
<head>
<title>Package Metrics Dashboard</title>
+ <link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.css" />
+ <style>
+ body {{
+ background-color: #f5f6fa;
+ margin: 0;
+ padding: 20px;
+ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
+ }}
+
+ .date-picker-container {{
+ background: white;
+ padding: 15px;
+ border-radius: 12px;
+ box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+ margin: 20px auto;
+ width: fit-content;
+ }}
+
+ #daterange {{
+ padding: 8px 12px;
+ border: 1px solid #ddd;
+ border-radius: 8px;
+ font-size: 14px;
+ width: 300px;
+ cursor: pointer;
+ }}
+
+ .quick-ranges {{
+ margin-top: 10px;
+ display: flex;
+ gap: 8px;
+ justify-content: center;
+ }}
+
+ .quick-ranges button {{
+ padding: 8px 16px;
+ border: 1px solid #e1e4e8;
+ border-radius: 8px;
+ background: white;
+ cursor: pointer;
+ font-size: 13px;
+ transition: all 0.2s ease;
+ }}
+
+ .quick-ranges button:hover {{
+ background: #f0f0f0;
+ transform: translateY(-1px);
+ }}
+
+ .dashboard-grid {{
+ display: grid;
+ grid-template-columns: 300px 1fr;
+ gap: 20px;
+ margin-top: 20px;
+ }}
+
+ .chart-container {{
+ background: white;
+ border-radius: 12px;
+ box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+ padding: 20px;
+ height: 350px;
+ }}
+
+ .chart-row {{
+ display: grid;
+ grid-template-columns: repeat(2, 1fr);
+ gap: 20px;
+ }}
+
+ .chart-box {{
+ background: white;
+ border-radius: 12px;
+ box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+ padding: 20px;
+ display: flex;
+ flex-direction: column;
+ }}
+
+ .chart-title {{
+ font-size: 16px;
+ font-weight: 600;
+ color: #2c3e50;
+ margin-bottom: 15px;
+ padding-bottom: 10px;
+ border-bottom: 1px solid #eee;
+ }}
+
+ .status-container {{
+ background: white;
+ border-radius: 12px;
+ box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+ padding: 20px;
+ height: 350px;
+ display: flex;
+ flex-direction: column;
+ justify-content: center;
+ }}
+
+ /* Override Plotly's default margins */
+ .js-plotly-plot .plotly {{
+ margin: 0 !important;
+ }}
+ </style>
</head>
<body>
- <div id="dashboard">
- {fig.to_html(include_plotlyjs=True, full_html=False)}
+ <div class="date-picker-container">
+ <input type="text" id="daterange" />
+ <div class="quick-ranges">
+ <button onclick="setQuickRange('1h')">Last Hour</button>
+ <button onclick="setQuickRange('6h')">Last 6 Hours</button>
+ <button onclick="setQuickRange('1d')">Last 24 Hours</button>
+ <button onclick="setQuickRange('7d')">Last 7 Days</button>
+ <button onclick="setQuickRange('30d')">Last 30 Days</button>
+ <button onclick="setQuickRange('all')">All Time</button>
+ </div>
+ </div>
+
+ <div class="dashboard-grid">
+ <div class="status-container">
+ <div class="chart-title">Test Status</div>
+ <div id="status-chart"></div>
+ </div>
+ <div class="chart-row">
+ <div class="chart-box">
+ <div class="chart-title">Package Size Trend</div>
+ <div id="size-chart"></div>
+ </div>
+ <div class="chart-box">
+ <div class="chart-title">Line Count Trend</div>
+ <div id="lines-chart"></div>
+ </div>
+ </div>
+ <div class="chart-row">
+ <div class="chart-box">
+ <div class="chart-title">Tokens per Second</div>
+ <div id="tokens-chart"></div>
+ </div>
+ </div>
</div>
+
+ <script type="text/javascript" src="https://cdn.jsdelivr.net/jquery/latest/jquery.min.js"></script>
+ <script type="text/javascript" src="https://cdn.jsdelivr.net/momentjs/latest/moment.min.js"></script>
+ <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.min.js"></script>
+ <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
<script>
- const plot = document.getElementById('dashboard').getElementsByClassName('plotly-graph-div')[0];
- plot.on('plotly_click', function(data) {{
- const point = data.points[0];
- const commitUrl = point.customdata[1];
- window.open(commitUrl, '_blank');
+ let globalMinDate = null;
+ let globalMaxDate = null;
+
+ // Split the original figure into separate charts
+ const originalData = {fig.to_json()};
+
+ function initializeCharts() {{
+ // Create the status indicator
+ if (originalData.data[0].type === 'indicator') {{
+ Plotly.newPlot('status-chart',
+ [originalData.data[0]],
+ {{
+ ...originalData.layout,
+ margin: {{ t: 0, b: 0, l: 0, r: 0 }},
+ height: 280
+ }}
+ );
+ }}
+
+ // Create the size trend chart
+ const sizeTrace = originalData.data.find(trace => trace.name === 'Package Size');
+ if (sizeTrace) {{
+ Plotly.newPlot('size-chart',
+ [sizeTrace],
+ {{
+ showlegend: false,
+ height: 280,
+ margin: {{ t: 10, b: 40, l: 50, r: 20 }},
+ yaxis: {{ title: 'Size (MB)' }},
+ xaxis: {{
+ type: 'date',
+ title: null,
+ range: [sizeTrace.x[0], sizeTrace.x[sizeTrace.x.length - 1]]
+ }}
+ }}
+ );
+ }}
+
+ // Create the line count chart
+ const lineTrace = originalData.data.find(trace => trace.name === 'Line Count');
+ if (lineTrace) {{
+ Plotly.newPlot('lines-chart',
+ [lineTrace],
+ {{
+ showlegend: false,
+ height: 280,
+ margin: {{ t: 10, b: 40, l: 50, r: 20 }},
+ yaxis: {{ title: 'Total Lines' }},
+ xaxis: {{
+ type: 'date',
+ title: null,
+ range: [lineTrace.x[0], lineTrace.x[lineTrace.x.length - 1]]
+ }}
+ }}
+ );
+ }}
+
+ // Create the tokens per second chart
+ const tokensTrace = originalData.data.find(trace => trace.name === 'Tokens/Second');
+ if (tokensTrace) {{
+ Plotly.newPlot('tokens-chart',
+ [tokensTrace],
+ {{
+ showlegend: false,
+ height: 280,
+ margin: {{ t: 10, b: 40, l: 50, r: 20 }},
+ yaxis: {{ title: 'Tokens/Second' }},
+ xaxis: {{
+ type: 'date',
+ title: null,
+ range: [tokensTrace.x[0], tokensTrace.x[tokensTrace.x.length - 1]]
+ }}
+ }}
+ );
+ }}
+
+ // Add debug logs to check axis names
+ console.log('Size Chart Layout:', document.getElementById('size-chart').layout);
+ console.log('Lines Chart Layout:', document.getElementById('lines-chart').layout);
+ console.log('Tokens Chart Layout:', document.getElementById('tokens-chart').layout);
+ }}
+
+ function setQuickRange(range) {{
+ let start, end = moment();
+
+ switch(range) {{
+ case '1h':
+ start = moment().subtract(1, 'hours');
+ break;
+ case '6h':
+ start = moment().subtract(6, 'hours');
+ break;
+ case '1d':
+ start = moment().subtract(1, 'days');
+ break;
+ case '7d':
+ start = moment().subtract(7, 'days');
+ break;
+ case '30d':
+ start = moment().subtract(30, 'days');
+ break;
+ case 'all':
+ start = moment(globalMinDate);
+ end = moment(globalMaxDate);
+ break;
+ }}
+
+ $('#daterange').data('daterangepicker').setStartDate(start);
+ $('#daterange').data('daterangepicker').setEndDate(end);
+ updatePlotRange(start.toISOString(), end.toISOString());
+ }}
+
+ function updatePlotRange(startDate, endDate) {{
+ console.log('Updating range:', startDate, endDate);
+
+ // Get the actual x-axis names from the chart layouts
+ const sizeChartLayout = document.getElementById('size-chart').layout;
+ const sizeXAxisName = Object.keys(sizeChartLayout).find(key => key.startsWith('xaxis'));
+
+ const linesChartLayout = document.getElementById('lines-chart').layout;
+ const linesXAxisName = Object.keys(linesChartLayout).find(key => key.startsWith('xaxis'));
+
+ const tokensChartLayout = document.getElementById('tokens-chart').layout;
+ const tokensXAxisName = Object.keys(tokensChartLayout).find(key => key.startsWith('xaxis'));
+
+ // Update the ranges
+ const sizeUpdateLayout = {{}};
+ sizeUpdateLayout[`${{sizeXAxisName}}.range`] = [startDate, endDate];
+
+ const linesUpdateLayout = {{}};
+ linesUpdateLayout[`${{linesXAxisName}}.range`] = [startDate, endDate];
+
+ const tokensUpdateLayout = {{}};
+ tokensUpdateLayout[`${{tokensXAxisName}}.range`] = [startDate, endDate];
+
+ // Update both charts
+ Plotly.relayout('size-chart', sizeUpdateLayout)
+ .catch(err => console.error('Error updating size chart:', err));
+
+ Plotly.relayout('lines-chart', linesUpdateLayout)
+ .catch(err => console.error('Error updating lines chart:', err));
+
+ Plotly.relayout('tokens-chart', tokensUpdateLayout)
+ .catch(err => console.error('Error updating tokens chart:', err));
+ }}
+
+ function findDateRange(data) {{
+ let minDate = null;
+ let maxDate = null;
+
+ data.forEach(trace => {{
+ if (trace.x && trace.x.length > 0) {{
+ const dates = trace.x.map(d => new Date(d));
+ const traceMin = new Date(Math.min(...dates));
+ const traceMax = new Date(Math.max(...dates));
+
+ if (!minDate || traceMin < minDate) minDate = traceMin;
+ if (!maxDate || traceMax > maxDate) maxDate = traceMax;
+ }}
+ }});
+
+ return {{ minDate, maxDate }};
+ }}
+
+ // Initialize everything when document is ready
+ $(document).ready(function() {{
+ // Initialize charts
+ initializeCharts();
+
+ // Find date range from data
+ const {{ minDate, maxDate }} = findDateRange(originalData.data);
+ globalMinDate = minDate;
+ globalMaxDate = maxDate;
+
+ // Initialize daterangepicker
+ $('#daterange').daterangepicker({{
+ startDate: minDate,
+ endDate: maxDate,
+ minDate: minDate,
+ maxDate: maxDate,
+ timePicker: true,
+ timePicker24Hour: true,
+ timePickerIncrement: 1,
+ opens: 'center',
+ locale: {{
+ format: 'YYYY-MM-DD HH:mm',
+ applyLabel: "Apply",
+ cancelLabel: "Cancel",
+ customRangeLabel: "Custom Range"
+ }},
+ ranges: {{
+ 'Last Hour': [moment().subtract(1, 'hours'), moment()],
+ 'Last 6 Hours': [moment().subtract(6, 'hours'), moment()],
+ 'Last 24 Hours': [moment().subtract(1, 'days'), moment()],
+ 'Last 7 Days': [moment().subtract(7, 'days'), moment()],
+ 'Last 30 Days': [moment().subtract(30, 'days'), moment()],
+ 'All Time': [moment(minDate), moment(maxDate)]
+ }}
+ }});
+
+ // Update plots when date range changes
+ $('#daterange').on('apply.daterangepicker', function(ev, picker) {{
+ console.log('Date range changed:', picker.startDate.toISOString(), picker.endDate.toISOString());
+ updatePlotRange(picker.startDate.toISOString(), picker.endDate.toISOString());
+ }});
+
+ // Add click handlers for charts
+ ['size-chart', 'lines-chart', 'tokens-chart'].forEach(chartId => {{
+ const chart = document.getElementById(chartId);
+ if (chart) {{
+ chart.on('plotly_click', function(data) {{
+ const point = data.points[0];
+ if (point.customdata && point.customdata[1]) {{
+ window.open(point.customdata[1], '_blank');
+ }}
+ }});
+ }}
+ }});
+
+ // Add debug logging for chart initialization
+ console.log('Size Chart:', document.getElementById('size-chart'));
+ console.log('Lines Chart:', document.getElementById('lines-chart'));
+ console.log('Tokens Chart:', document.getElementById('tokens-chart'));
}});
</script>
</body>
@@ -336,6 +828,21 @@ class PackageSizeTracker:
'linecount_change': linecount_change
})
+ if not df_benchmark.empty:
+ latest = df_benchmark.iloc[-1]
+ previous = df_benchmark.iloc[-2] if len(df_benchmark) > 1 else latest
+ tokens_change = float(latest['tokens_per_second'] - previous['tokens_per_second'])
+ if not latest_data: # Only add timestamp and commit info if not already added
+ latest_data.update({
+ 'timestamp': latest['timestamp'].isoformat(),
+ 'commit_hash': latest['commit_hash'],
+ 'commit_url': latest['commit_url'],
+ })
+ latest_data.update({
+ 'tokens_per_second': float(latest['tokens_per_second']),
+ 'tokens_change': tokens_change
+ })
+
if latest_data:
with open(output_dir / 'latest_data.json', 'w') as f:
json.dump(latest_data, f, indent=2)
@@ -370,6 +877,11 @@ class PackageSizeTracker:
change_symbol = "↓" if change <= 0 else "↑"
print(f"Change: {change_symbol} {abs(change):,}")
+ if 'tokens_per_second' in latest_data:
+ print("\nBenchmark Stats:")
+ print(f"Tokens per Second: {latest_data['tokens_per_second']:.2f}")
+ print(f"Time to First Token: {latest_data['time_to_first_token']:.3f}s")
+
print("\n")
async def main():
← ab333ea5 circle tail logs
·
back to Exo
·
dashboard fix limit 062f5e3e →