Browse Source
[tools/test] Introduce fps comparassion tool
[tools/test] Introduce fps comparassion tool
* It should be really improved, but it's a start Signed-off-by: Caio Oliveira <caiooliveirafarias0@gmail.com>pull/2532/head
Caio Oliveira
6 months ago
No known key found for this signature in database
GPG Key ID: 362DA3DC1901E080
2 changed files with 225 additions and 0 deletions
@ -0,0 +1,91 @@ |
|||
#!/bin/bash -e |
|||
|
|||
# SPDX-FileCopyrightText: Copyright 2025 DraVee |
|||
# SPDX-License-Identifier: GPL-3.0-or-later |
|||
|
|||
# Usage/help |
|||
show_help() { |
|||
echo "Usage: $0 [--temp | <log_folder>]" |
|||
echo |
|||
echo "Options:" |
|||
echo " --temp Use a temporary folder (mktemp) for logs" |
|||
echo " <log_folder> Use the specified folder for logs" |
|||
echo " -h, --help Show this help message" |
|||
} |
|||
|
|||
# Parse arguments |
|||
if [[ "$1" == "-h" || "$1" == "--help" ]]; then |
|||
show_help |
|||
exit 0 |
|||
fi |
|||
|
|||
if [[ "$1" == "--temp" ]]; then |
|||
BASE_LOG_DIR=$(mktemp -d) |
|||
echo "Using temporary log folder: $BASE_LOG_DIR" |
|||
elif [[ -n "$1" ]]; then |
|||
BASE_LOG_DIR="$1" |
|||
else |
|||
BASE_LOG_DIR="$HOME/.cache/test-logs" |
|||
fi |
|||
|
|||
mkdir -p "$BASE_LOG_DIR" |
|||
|
|||
# Check required programs |
|||
for cmd in python3 mangohud find; do |
|||
if ! command -v "$cmd" >/dev/null 2>&1; then |
|||
echo "Error: $cmd is not installed. Please install it before running this script." |
|||
exit 1 |
|||
fi |
|||
done |
|||
|
|||
# Log duration |
|||
WAIT_DURATION=5 |
|||
LOG_DURATION=$((30 + WAIT_DURATION)) |
|||
|
|||
# Loop through all build*/bin/eden executables |
|||
for eden_bin in build*/bin/eden; do |
|||
if [[ ! -x "$eden_bin" ]]; then |
|||
echo "Skipping $eden_bin: not executable" |
|||
continue |
|||
fi |
|||
|
|||
# Extract build name |
|||
build_name=$(dirname "$eden_bin" | cut -d'/' -f1) |
|||
|
|||
# Timestamp for unique log folder |
|||
timestamp=$(date +%Y-%m-%d_%H-%M-%S) |
|||
log_dir="$BASE_LOG_DIR/$build_name/$timestamp" |
|||
mkdir -p "$log_dir" |
|||
|
|||
echo "Running $eden_bin → logs will be saved in $log_dir" |
|||
|
|||
# Set MangoHud environment variables |
|||
export MANGOHUD=1 |
|||
export MANGOHUD_LOG=1 |
|||
export MANGOHUD_CONFIG="output_folder=$log_dir;log_duration=$LOG_DURATION;autostart_log=$WAIT_DURATION" |
|||
|
|||
# Run Eden in background and capture its PID |
|||
QT_QPA_PLATFORM=xcb "$eden_bin" & |
|||
EDEN_PID=$! |
|||
|
|||
# Monitor MangoHud logs in real time for _summary.csv creation |
|||
summary_file="" |
|||
while [[ ! -f "$summary_file" ]]; do |
|||
summary_file=$(find "$log_dir" -name "*_summary.csv" | head -n 1) |
|||
sleep 0.5 |
|||
done |
|||
|
|||
echo "Summary detected: $summary_file" |
|||
echo "Stopping $eden_bin..." |
|||
|
|||
# Kill the Eden process |
|||
kill "$EDEN_PID" |
|||
sleep 5 |
|||
kill -9 "$EDEN_PID" 2>/dev/null || true |
|||
wait "$EDEN_PID" 2>/dev/null || true |
|||
done |
|||
|
|||
# Run comparison script |
|||
echo "All builds finished. Running compare_logs.py..." |
|||
python3 tools/test/compare_logs.py "$BASE_LOG_DIR" |
|||
|
|||
@ -0,0 +1,134 @@ |
|||
#!/usr/bin/env python3 |
|||
|
|||
# SPDX-FileCopyrightText: Copyright 2025 DraVee |
|||
# SPDX-License-Identifier: GPL-3.0-or-later |
|||
|
|||
import sys |
|||
import pandas as pd |
|||
import matplotlib.pyplot as plt |
|||
import glob |
|||
import os |
|||
|
|||
# Check required Python modules |
|||
required_modules = ["pandas", "matplotlib"] |
|||
missing_modules = [] |
|||
|
|||
for mod in required_modules: |
|||
try: |
|||
__import__(mod) |
|||
except ImportError: |
|||
missing_modules.append(mod) |
|||
|
|||
if missing_modules: |
|||
print(f"Error: Missing required Python modules: {', '.join(missing_modules)}") |
|||
print("Please install them, e.g.:") |
|||
print(f" python3 -m pip install {' '.join(missing_modules)}") |
|||
sys.exit(1) |
|||
|
|||
# Get log folder from command-line argument |
|||
if len(sys.argv) < 2: |
|||
print("Usage: python3 compare_logs.py <log_folder>") |
|||
sys.exit(1) |
|||
|
|||
log_base_folder = os.path.expanduser(sys.argv[1]) |
|||
if not os.path.isdir(log_base_folder): |
|||
print(f"Error: '{log_base_folder}' is not a valid folder") |
|||
sys.exit(1) |
|||
|
|||
# Find all CSV files recursively (ignore summary CSVs) |
|||
csv_files = sorted(glob.glob(os.path.join(log_base_folder, "**/eden_*.csv"), recursive=True)) |
|||
csv_files = [f for f in csv_files if not f.endswith("_summary.csv")] |
|||
|
|||
if not csv_files: |
|||
print(f"No CSV files found in {log_base_folder} or its subfolders") |
|||
sys.exit(0) |
|||
|
|||
# Prepare plotting |
|||
plt.figure(figsize=(14, 7)) |
|||
colors = plt.colormaps['tab10'] |
|||
|
|||
stats = [] |
|||
|
|||
# Track which folders have CSVs |
|||
folders_with_csv = set() |
|||
|
|||
for i, csv_file in enumerate(csv_files): |
|||
folder = os.path.dirname(csv_file) |
|||
folders_with_csv.add(folder) |
|||
|
|||
# Corresponding summary file |
|||
summary_file = csv_file.replace(".csv", "_summary.csv") |
|||
|
|||
# Skip empty CSVs |
|||
if os.path.getsize(csv_file) == 0: |
|||
print(f"Skipping {csv_file}: file is empty") |
|||
continue |
|||
|
|||
# Read main CSV (skip system info lines) |
|||
df = pd.read_csv(csv_file, skiprows=2) |
|||
df.columns = df.columns.str.strip() |
|||
|
|||
if 'fps' not in df.columns: |
|||
print(f"Skipping {csv_file}: no 'fps' column found") |
|||
continue |
|||
|
|||
y = df['fps'] |
|||
x = range(len(y)) |
|||
|
|||
# Compute statistics from main CSV |
|||
mean_fps = y.mean() |
|||
min_fps = y.min() |
|||
max_fps = y.max() |
|||
|
|||
# Read summary CSV if exists |
|||
summary_text = "" |
|||
if os.path.exists(summary_file): |
|||
try: |
|||
df_sum = pd.read_csv(summary_file) |
|||
avg = float(df_sum['Average FPS'][0]) |
|||
p0_1 = float(df_sum['0.1% Min FPS'][0]) |
|||
p1 = float(df_sum['1% Min FPS'][0]) |
|||
p97 = float(df_sum['97% Percentile FPS'][0]) |
|||
summary_text = f" | summary avg={avg:.1f}, 0.1%={p0_1:.1f}, 1%={p1:.1f}, 97%={p97:.1f}" |
|||
except Exception as e: |
|||
print(f"Could not read summary for {summary_file}: {e}") |
|||
|
|||
stats.append((os.path.basename(csv_file), mean_fps, min_fps, max_fps, summary_text)) |
|||
|
|||
# Plot FPS line with summary info |
|||
plt.plot(x, y, label=f"{os.path.basename(csv_file)} (avg={mean_fps:.1f}){summary_text}", color=colors(i % 10)) |
|||
|
|||
# Configure plot |
|||
plt.xlabel('Frame') |
|||
plt.ylabel('FPS') |
|||
plt.title('FPS Comparison Across All Builds') |
|||
plt.legend() |
|||
plt.grid(True) |
|||
plt.tight_layout() |
|||
|
|||
# Save plot |
|||
png_file = os.path.join(os.getcwd(), "fps_comparison_all_builds.png") |
|||
plt.savefig(png_file, dpi=200) |
|||
plt.show() |
|||
|
|||
# Print statistics in terminal |
|||
print("\nFPS Summary by file:") |
|||
for name, mean_fps, min_fps, max_fps, summary_text in stats: |
|||
print(f"{name}: mean={mean_fps:.1f}, min={min_fps:.1f}, max={max_fps:.1f}{summary_text}") |
|||
|
|||
print("\n----------------------------") |
|||
print(f"Total CSV files processed: {len(stats)}") |
|||
|
|||
# Track build folders (without timestamps) |
|||
build_folders = set() |
|||
for csv_file in csv_files: |
|||
run_folder = os.path.dirname(csv_file) |
|||
build_folder = os.path.dirname(run_folder) |
|||
build_folders.add(build_folder) |
|||
|
|||
print("\nBuild folders containing CSVs:") |
|||
for folder in sorted(build_folders): |
|||
print(f" - {folder}") |
|||
|
|||
print(f"Graph saved as: {png_file}") |
|||
|
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue