← back to Exo
include total line count in dashboard
e3dc3b2023981b3544513c13b988f638cea75d8d · 2024-11-25 13:39:47 +0400 · Alex Cheema
Files touched
M extra/dashboard/dashboard.py
Diff
commit e3dc3b2023981b3544513c13b988f638cea75d8d
Author: Alex Cheema <alexcheema123@gmail.com>
Date: Mon Nov 25 13:39:47 2024 +0400
include total line count in dashboard
---
extra/dashboard/dashboard.py | 307 ++++++++++++++++++++++++++++---------------
1 file changed, 202 insertions(+), 105 deletions(-)
diff --git a/extra/dashboard/dashboard.py b/extra/dashboard/dashboard.py
index d2191a3a..8697d169 100644
--- a/extra/dashboard/dashboard.py
+++ b/extra/dashboard/dashboard.py
@@ -7,6 +7,8 @@ import pandas as pd
import plotly.express as px
from typing import List, Dict, Optional
from pathlib import Path
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
class AsyncCircleCIClient:
def __init__(self, token: str, project_slug: str):
@@ -24,7 +26,7 @@ class AsyncCircleCIClient:
response.raise_for_status()
return await response.json()
- async def get_recent_pipelines(self, session: aiohttp.ClientSession, limit: int = 50) -> List[Dict]:
+ async def get_recent_pipelines(self, session: aiohttp.ClientSession, limit: int = 100) -> List[Dict]:
self.logger.info(f"Fetching {limit} recent pipelines...")
url = f"{self.base_url}/project/{self.project_slug}/pipeline"
params = {"limit": limit * 2}
@@ -111,39 +113,69 @@ class PackageSizeTracker:
return None
jobs = await self.client.get_workflow_jobs(session, pipeline["id"])
+
+ # Get package size data
size_job = next(
(j for j in jobs if j["name"] == "measure_pip_sizes" and j["status"] == "success"),
None
)
- if not size_job:
- self.logger.debug(f"No measure_pip_sizes job found for pipeline {pipeline['id']}")
- return None
-
- artifacts = await self.client.get_artifacts(session, size_job["job_number"])
- size_report = next(
- (a for a in artifacts if a["path"].endswith("pip-sizes.json")),
+ # Get line count data
+ linecount_job = next(
+ (j for j in jobs if j["name"] == "check_line_count" and j["status"] == "success"),
None
)
- if not size_report:
- self.logger.debug(f"No pip-sizes.json artifact found for job {size_job['job_number']}")
+ # Return None if no relevant jobs found
+ if not size_job and not linecount_job:
+ self.logger.debug(f"No relevant jobs found for pipeline {pipeline['id']}")
return None
- json_data = await self.client.get_json(session, size_report["url"])
data_point = {
"commit_hash": commit_info['commit_hash'],
"commit_url": commit_info['web_url'],
"timestamp": pipeline.get("created_at", pipeline.get("updated_at")),
- "total_size_mb": json_data["total_size_mb"],
- "packages": json_data["packages"]
}
- self.logger.info(
- f"Processed pipeline {pipeline['id']}: "
- f"commit {commit_info['commit_hash'][:7]}, "
- f"size {json_data['total_size_mb']:.2f}MB"
- )
+ # Process size data if available
+ if size_job:
+ size_artifacts = await self.client.get_artifacts(session, size_job["job_number"])
+ size_report = next(
+ (a for a in size_artifacts if a["path"].endswith("pip-sizes.json")),
+ None
+ )
+ if size_report:
+ size_data = await self.client.get_json(session, size_report["url"])
+ data_point.update({
+ "total_size_mb": size_data["total_size_mb"],
+ "packages": size_data["packages"]
+ })
+ self.logger.info(
+ f"Processed size data for pipeline {pipeline['id']}: "
+ f"commit {commit_info['commit_hash'][:7]}, "
+ f"size {size_data['total_size_mb']:.2f}MB"
+ )
+
+ # Process linecount data if available
+ if linecount_job:
+ linecount_artifacts = await self.client.get_artifacts(session, linecount_job["job_number"])
+ linecount_report = next(
+ (a for a in linecount_artifacts if a["path"].endswith("line-count-snapshot.json")),
+ None
+ )
+ if linecount_report:
+ linecount_data = await self.client.get_json(session, linecount_report["url"])
+ data_point.update({
+ "total_lines": linecount_data["total_lines"],
+ "total_files": linecount_data["total_files"],
+ "files": linecount_data["files"]
+ })
+ self.logger.info(
+ f"Processed line count data for pipeline {pipeline['id']}: "
+ f"commit {commit_info['commit_hash'][:7]}, "
+ f"lines {linecount_data['total_lines']:,}"
+ )
+
return data_point
except Exception as e:
@@ -154,7 +186,7 @@ class PackageSizeTracker:
self.logger.info("Starting data collection...")
async with aiohttp.ClientSession(headers=self.client.headers) as session:
# Get pipelines
- pipelines = await self.client.get_recent_pipelines(session, 50)
+ pipelines = await self.client.get_recent_pipelines(session, 100)
# Process all pipelines in parallel
tasks = [self.process_pipeline(session, pipeline) for pipeline in pipelines]
@@ -171,108 +203,173 @@ class PackageSizeTracker:
self.logger.error("No data to generate report from!")
return None
- df = pd.DataFrame(data)
- df['timestamp'] = pd.to_datetime(df['timestamp'])
- df = df.sort_values('timestamp')
- # commit_url is already in the data from process_pipeline
-
- # Create trend plot with updated styling
- fig = px.line(
- df,
- x='timestamp',
- y='total_size_mb',
- title='Package Size Trend',
- markers=True,
- hover_data={'commit_hash': True, 'timestamp': True, 'total_size_mb': ':.2f'},
- custom_data=['commit_hash', 'commit_url']
- )
- fig.update_layout(
- xaxis_title="Date",
- yaxis_title="Total Size (MB)",
- hovermode='x unified',
- plot_bgcolor='white',
- paper_bgcolor='white',
- font=dict(size=12),
- title_x=0.5,
- )
- fig.update_traces(
- line=dict(width=2),
- marker=dict(size=8),
- hovertemplate="<br>".join([
- "Commit: %{customdata[0]}",
- "Size: %{y:.2f}MB",
- "Date: %{x}",
- "<extra>Click to view commit</extra>"
- ])
- )
-
- # Add JavaScript for click handling
- fig.update_layout(
- clickmode='event',
- annotations=[
- dict(
- text="Click any point to view the commit on GitHub",
- xref="paper", yref="paper",
- x=0, y=1.05,
- showarrow=False
- )
- ]
- )
+ # Create separate dataframes for each metric
+ df_size = pd.DataFrame([d for d in data if 'total_size_mb' in d])
+ df_lines = pd.DataFrame([d for d in data if 'total_lines' in d])
# Ensure output directory exists
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
- # Save plot
- plot_path = output_dir / "package_size_trend.html"
- fig.write_html(
- str(plot_path),
- include_plotlyjs=True,
- full_html=True,
- post_script="""
- const plot = document.getElementsByClassName('plotly-graph-div')[0];
- plot.on('plotly_click', function(data) {
- const point = data.points[0];
- const commitUrl = point.customdata[1];
- window.open(commitUrl, '_blank');
- });
- """
+ # Create a single figure with subplots
+ fig = make_subplots(
+ rows=2, cols=1,
+ subplot_titles=('Package Size Trend', 'Line Count Trend'),
+ vertical_spacing=0.2
)
- # Generate summary
- latest = df.iloc[-1]
- previous = df.iloc[-2] if len(df) > 1 else latest
- size_change = latest['total_size_mb'] - previous['total_size_mb']
-
- latest_data = {
- 'timestamp': latest['timestamp'].isoformat(),
- 'commit_hash': latest['commit_hash'],
- 'total_size_mb': latest['total_size_mb'],
- 'size_change_mb': size_change,
- 'packages': latest['packages']
- }
+ # Add package size trace if we have data
+ if not df_size.empty:
+ df_size['timestamp'] = pd.to_datetime(df_size['timestamp'])
+ df_size = df_size.sort_values('timestamp')
+
+ fig.add_trace(
+ go.Scatter(
+ x=df_size['timestamp'],
+ y=df_size['total_size_mb'],
+ mode='lines+markers',
+ name='Package Size',
+ customdata=df_size[['commit_hash', 'commit_url']].values,
+ hovertemplate="<br>".join([
+ "Size: %{y:.2f}MB",
+ "Date: %{x}",
+ "Commit: %{customdata[0]}",
+ "<extra></extra>"
+ ])
+ ),
+ row=1, col=1
+ )
+ fig.update_yaxes(title_text="Size (MB)", row=1, col=1)
+
+ # Add line count trace if we have data
+ if not df_lines.empty:
+ df_lines['timestamp'] = pd.to_datetime(df_lines['timestamp'])
+ df_lines = df_lines.sort_values('timestamp')
+
+ fig.add_trace(
+ go.Scatter(
+ x=df_lines['timestamp'],
+ y=df_lines['total_lines'],
+ mode='lines+markers',
+ name='Line Count',
+ customdata=df_lines[['commit_hash', 'commit_url']].values,
+ hovertemplate="<br>".join([
+ "Lines: %{y:,.0f}",
+ "Date: %{x}",
+ "Commit: %{customdata[0]}",
+ "<extra></extra>"
+ ])
+ ),
+ row=2, col=1
+ )
+ fig.update_yaxes(title_text="Total Lines", row=2, col=1)
- with open(output_dir / 'latest_data.json', 'w') as f:
- json.dump(latest_data, f, indent=2)
+ # Update layout
+ fig.update_layout(
+ height=800, # Taller to accommodate both plots
+ showlegend=False,
+ title_text="Package Metrics Dashboard",
+ title_x=0.5,
+ plot_bgcolor='white',
+ paper_bgcolor='white',
+ font=dict(size=12),
+ hovermode='x unified',
+ xaxis=dict(title_text="Date"),
+ xaxis2=dict(title_text="Date")
+ )
- self._print_summary(latest_data)
- self.logger.info(f"Report generated in {output_dir}")
- return str(plot_path)
+ # Add click event handling
+ dashboard_html = f"""
+ <html>
+ <head>
+ <title>Package Metrics Dashboard</title>
+ </head>
+ <body>
+ <div id="dashboard">
+ {fig.to_html(include_plotlyjs=True, full_html=False)}
+ </div>
+ <script>
+ const plot = document.getElementById('dashboard').getElementsByClassName('plotly-graph-div')[0];
+ plot.on('plotly_click', function(data) {{
+ const point = data.points[0];
+ const commitUrl = point.customdata[1];
+ window.open(commitUrl, '_blank');
+ }});
+ </script>
+ </body>
+ </html>
+ """
+
+ # Write the dashboard
+ dashboard_path = output_dir / "dashboard.html"
+ with open(dashboard_path, "w") as f:
+ f.write(dashboard_html)
+
+ # Generate summary with available metrics
+ latest_data = {}
+
+ if not df_size.empty:
+ latest = df_size.iloc[-1]
+ previous = df_size.iloc[-2] if len(df_size) > 1 else latest
+ size_change = latest['total_size_mb'] - previous['total_size_mb']
+ latest_data.update({
+ 'timestamp': latest['timestamp'].isoformat(),
+ 'commit_hash': latest['commit_hash'],
+ 'commit_url': latest['commit_url'],
+ 'total_size_mb': latest['total_size_mb'],
+ 'size_change_mb': size_change,
+ 'packages': latest.get('packages', [])
+ })
+
+ if not df_lines.empty:
+ latest = df_lines.iloc[-1]
+ previous = df_lines.iloc[-2] if len(df_lines) > 1 else latest
+ linecount_change = latest['total_lines'] - previous['total_lines']
+ if not latest_data: # Only add timestamp and commit info if not already added
+ latest_data.update({
+ 'timestamp': latest['timestamp'].isoformat(),
+ 'commit_hash': latest['commit_hash'],
+ 'commit_url': latest['commit_url'],
+ })
+ latest_data.update({
+ 'total_lines': latest['total_lines'],
+ 'linecount_change': linecount_change
+ })
+
+ if latest_data:
+ with open(output_dir / 'latest_data.json', 'w') as f:
+ json.dump(latest_data, f, indent=2)
+
+ self._print_summary(latest_data)
+ self.logger.info(f"Report generated in {output_dir}")
+ return str(output_dir)
+
+ return None
def _print_summary(self, latest_data: Dict):
print("\n=== Package Size Summary ===")
print(f"Timestamp: {latest_data['timestamp']}")
print(f"Commit: {latest_data['commit_hash'][:7]}")
- print(f"Total Size: {latest_data['total_size_mb']:.2f}MB")
- change = latest_data['size_change_mb']
- change_symbol = "↓" if change <= 0 else "↑"
- print(f"Change: {change_symbol} {abs(change):.2f}MB")
+ if 'total_size_mb' in latest_data:
+ print(f"Total Size: {latest_data['total_size_mb']:.2f}MB")
+ change = latest_data['size_change_mb']
+ change_symbol = "↓" if change <= 0 else "↑"
+ print(f"Change: {change_symbol} {abs(change):.2f}MB")
+
+ if latest_data.get('packages'):
+ print("\nTop 5 Largest Packages:")
+ sorted_packages = sorted(latest_data['packages'], key=lambda x: x['size_mb'], reverse=True)
+ for pkg in sorted_packages[:5]:
+ print(f"- {pkg['name']}: {pkg['size_mb']:.2f}MB")
+
+ if 'total_lines' in latest_data:
+ print("\nLine Count Stats:")
+ print(f"Total Lines: {latest_data['total_lines']:,}")
+ change = latest_data['linecount_change']
+ change_symbol = "↓" if change <= 0 else "↑"
+ print(f"Change: {change_symbol} {abs(change):,}")
- print("\nTop 5 Largest Packages:")
- sorted_packages = sorted(latest_data['packages'], key=lambda x: x['size_mb'], reverse=True)
- for pkg in sorted_packages[:5]:
- print(f"- {pkg['name']}: {pkg['size_mb']:.2f}MB")
print("\n")
async def main():
← ea9b0436 fix indent
·
back to Exo
·
dashboard fix int64 28de4fd7 →