Browse Source
[tools/test] Introduce fps comparassion tool
[tools/test] Introduce fps comparassion tool
* It should be really improved, but it's a start Signed-off-by: Caio Oliveira <caiooliveirafarias0@gmail.com>pull/2532/head
Caio Oliveira
6 months ago
No known key found for this signature in database
GPG Key ID: 362DA3DC1901E080
2 changed files with 225 additions and 0 deletions
@ -0,0 +1,91 @@ |
|||||
|
#!/bin/bash -e |
||||
|
|
||||
|
# SPDX-FileCopyrightText: Copyright 2025 DraVee |
||||
|
# SPDX-License-Identifier: GPL-3.0-or-later |
||||
|
|
||||
|
# Usage/help |
||||
|
show_help() { |
||||
|
echo "Usage: $0 [--temp | <log_folder>]" |
||||
|
echo |
||||
|
echo "Options:" |
||||
|
echo " --temp Use a temporary folder (mktemp) for logs" |
||||
|
echo " <log_folder> Use the specified folder for logs" |
||||
|
echo " -h, --help Show this help message" |
||||
|
} |
||||
|
|
||||
|
# Parse arguments |
||||
|
if [[ "$1" == "-h" || "$1" == "--help" ]]; then |
||||
|
show_help |
||||
|
exit 0 |
||||
|
fi |
||||
|
|
||||
|
if [[ "$1" == "--temp" ]]; then |
||||
|
BASE_LOG_DIR=$(mktemp -d) |
||||
|
echo "Using temporary log folder: $BASE_LOG_DIR" |
||||
|
elif [[ -n "$1" ]]; then |
||||
|
BASE_LOG_DIR="$1" |
||||
|
else |
||||
|
BASE_LOG_DIR="$HOME/.cache/test-logs" |
||||
|
fi |
||||
|
|
||||
|
mkdir -p "$BASE_LOG_DIR" |
||||
|
|
||||
|
# Check required programs |
||||
|
for cmd in python3 mangohud find; do |
||||
|
if ! command -v "$cmd" >/dev/null 2>&1; then |
||||
|
echo "Error: $cmd is not installed. Please install it before running this script." |
||||
|
exit 1 |
||||
|
fi |
||||
|
done |
||||
|
|
||||
|
# Log duration |
||||
|
WAIT_DURATION=5 |
||||
|
LOG_DURATION=$((30 + WAIT_DURATION)) |
||||
|
|
||||
|
# Loop through all build*/bin/eden executables |
||||
|
for eden_bin in build*/bin/eden; do |
||||
|
if [[ ! -x "$eden_bin" ]]; then |
||||
|
echo "Skipping $eden_bin: not executable" |
||||
|
continue |
||||
|
fi |
||||
|
|
||||
|
# Extract build name |
||||
|
build_name=$(dirname "$eden_bin" | cut -d'/' -f1) |
||||
|
|
||||
|
# Timestamp for unique log folder |
||||
|
timestamp=$(date +%Y-%m-%d_%H-%M-%S) |
||||
|
log_dir="$BASE_LOG_DIR/$build_name/$timestamp" |
||||
|
mkdir -p "$log_dir" |
||||
|
|
||||
|
echo "Running $eden_bin → logs will be saved in $log_dir" |
||||
|
|
||||
|
# Set MangoHud environment variables |
||||
|
export MANGOHUD=1 |
||||
|
export MANGOHUD_LOG=1 |
||||
|
export MANGOHUD_CONFIG="output_folder=$log_dir;log_duration=$LOG_DURATION;autostart_log=$WAIT_DURATION" |
||||
|
|
||||
|
# Run Eden in background and capture its PID |
||||
|
QT_QPA_PLATFORM=xcb "$eden_bin" & |
||||
|
EDEN_PID=$! |
||||
|
|
||||
|
# Monitor MangoHud logs in real time for _summary.csv creation |
||||
|
summary_file="" |
||||
|
while [[ ! -f "$summary_file" ]]; do |
||||
|
summary_file=$(find "$log_dir" -name "*_summary.csv" | head -n 1) |
||||
|
sleep 0.5 |
||||
|
done |
||||
|
|
||||
|
echo "Summary detected: $summary_file" |
||||
|
echo "Stopping $eden_bin..." |
||||
|
|
||||
|
# Kill the Eden process |
||||
|
kill "$EDEN_PID" |
||||
|
sleep 5 |
||||
|
kill -9 "$EDEN_PID" 2>/dev/null || true |
||||
|
wait "$EDEN_PID" 2>/dev/null || true |
||||
|
done |
||||
|
|
||||
|
# Run comparison script |
||||
|
echo "All builds finished. Running compare_logs.py..." |
||||
|
python3 tools/test/compare_logs.py "$BASE_LOG_DIR" |
||||
|
|
||||
@ -0,0 +1,134 @@ |
|||||
|
#!/usr/bin/env python3 |
||||
|
|
||||
|
# SPDX-FileCopyrightText: Copyright 2025 DraVee |
||||
|
# SPDX-License-Identifier: GPL-3.0-or-later |
||||
|
|
||||
|
import sys |
||||
|
import pandas as pd |
||||
|
import matplotlib.pyplot as plt |
||||
|
import glob |
||||
|
import os |
||||
|
|
||||
|
# Check required Python modules |
||||
|
required_modules = ["pandas", "matplotlib"] |
||||
|
missing_modules = [] |
||||
|
|
||||
|
for mod in required_modules: |
||||
|
try: |
||||
|
__import__(mod) |
||||
|
except ImportError: |
||||
|
missing_modules.append(mod) |
||||
|
|
||||
|
if missing_modules: |
||||
|
print(f"Error: Missing required Python modules: {', '.join(missing_modules)}") |
||||
|
print("Please install them, e.g.:") |
||||
|
print(f" python3 -m pip install {' '.join(missing_modules)}") |
||||
|
sys.exit(1) |
||||
|
|
||||
|
# Get log folder from command-line argument |
||||
|
if len(sys.argv) < 2: |
||||
|
print("Usage: python3 compare_logs.py <log_folder>") |
||||
|
sys.exit(1) |
||||
|
|
||||
|
log_base_folder = os.path.expanduser(sys.argv[1]) |
||||
|
if not os.path.isdir(log_base_folder): |
||||
|
print(f"Error: '{log_base_folder}' is not a valid folder") |
||||
|
sys.exit(1) |
||||
|
|
||||
|
# Find all CSV files recursively (ignore summary CSVs) |
||||
|
csv_files = sorted(glob.glob(os.path.join(log_base_folder, "**/eden_*.csv"), recursive=True)) |
||||
|
csv_files = [f for f in csv_files if not f.endswith("_summary.csv")] |
||||
|
|
||||
|
if not csv_files: |
||||
|
print(f"No CSV files found in {log_base_folder} or its subfolders") |
||||
|
sys.exit(0) |
||||
|
|
||||
|
# Prepare plotting |
||||
|
plt.figure(figsize=(14, 7)) |
||||
|
colors = plt.colormaps['tab10'] |
||||
|
|
||||
|
stats = [] |
||||
|
|
||||
|
# Track which folders have CSVs |
||||
|
folders_with_csv = set() |
||||
|
|
||||
|
for i, csv_file in enumerate(csv_files): |
||||
|
folder = os.path.dirname(csv_file) |
||||
|
folders_with_csv.add(folder) |
||||
|
|
||||
|
# Corresponding summary file |
||||
|
summary_file = csv_file.replace(".csv", "_summary.csv") |
||||
|
|
||||
|
# Skip empty CSVs |
||||
|
if os.path.getsize(csv_file) == 0: |
||||
|
print(f"Skipping {csv_file}: file is empty") |
||||
|
continue |
||||
|
|
||||
|
# Read main CSV (skip system info lines) |
||||
|
df = pd.read_csv(csv_file, skiprows=2) |
||||
|
df.columns = df.columns.str.strip() |
||||
|
|
||||
|
if 'fps' not in df.columns: |
||||
|
print(f"Skipping {csv_file}: no 'fps' column found") |
||||
|
continue |
||||
|
|
||||
|
y = df['fps'] |
||||
|
x = range(len(y)) |
||||
|
|
||||
|
# Compute statistics from main CSV |
||||
|
mean_fps = y.mean() |
||||
|
min_fps = y.min() |
||||
|
max_fps = y.max() |
||||
|
|
||||
|
# Read summary CSV if exists |
||||
|
summary_text = "" |
||||
|
if os.path.exists(summary_file): |
||||
|
try: |
||||
|
df_sum = pd.read_csv(summary_file) |
||||
|
avg = float(df_sum['Average FPS'][0]) |
||||
|
p0_1 = float(df_sum['0.1% Min FPS'][0]) |
||||
|
p1 = float(df_sum['1% Min FPS'][0]) |
||||
|
p97 = float(df_sum['97% Percentile FPS'][0]) |
||||
|
summary_text = f" | summary avg={avg:.1f}, 0.1%={p0_1:.1f}, 1%={p1:.1f}, 97%={p97:.1f}" |
||||
|
except Exception as e: |
||||
|
print(f"Could not read summary for {summary_file}: {e}") |
||||
|
|
||||
|
stats.append((os.path.basename(csv_file), mean_fps, min_fps, max_fps, summary_text)) |
||||
|
|
||||
|
# Plot FPS line with summary info |
||||
|
plt.plot(x, y, label=f"{os.path.basename(csv_file)} (avg={mean_fps:.1f}){summary_text}", color=colors(i % 10)) |
||||
|
|
||||
|
# Configure plot |
||||
|
plt.xlabel('Frame') |
||||
|
plt.ylabel('FPS') |
||||
|
plt.title('FPS Comparison Across All Builds') |
||||
|
plt.legend() |
||||
|
plt.grid(True) |
||||
|
plt.tight_layout() |
||||
|
|
||||
|
# Save plot |
||||
|
png_file = os.path.join(os.getcwd(), "fps_comparison_all_builds.png") |
||||
|
plt.savefig(png_file, dpi=200) |
||||
|
plt.show() |
||||
|
|
||||
|
# Print statistics in terminal |
||||
|
print("\nFPS Summary by file:") |
||||
|
for name, mean_fps, min_fps, max_fps, summary_text in stats: |
||||
|
print(f"{name}: mean={mean_fps:.1f}, min={min_fps:.1f}, max={max_fps:.1f}{summary_text}") |
||||
|
|
||||
|
print("\n----------------------------") |
||||
|
print(f"Total CSV files processed: {len(stats)}") |
||||
|
|
||||
|
# Track build folders (without timestamps) |
||||
|
build_folders = set() |
||||
|
for csv_file in csv_files: |
||||
|
run_folder = os.path.dirname(csv_file) |
||||
|
build_folder = os.path.dirname(run_folder) |
||||
|
build_folders.add(build_folder) |
||||
|
|
||||
|
print("\nBuild folders containing CSVs:") |
||||
|
for folder in sorted(build_folders): |
||||
|
print(f" - {folder}") |
||||
|
|
||||
|
print(f"Graph saved as: {png_file}") |
||||
|
|
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue