diff --git a/tools/test/compare_builds.sh b/tools/test/compare_builds.sh index 788558d7e0..54593af6c4 100755 --- a/tools/test/compare_builds.sh +++ b/tools/test/compare_builds.sh @@ -40,7 +40,7 @@ done # Log duration WAIT_DURATION=5 -LOG_DURATION=$((30 + WAIT_DURATION)) +LOG_DURATION=$((60 + WAIT_DURATION)) # Loop through all build*/bin/eden executables for eden_bin in build*/bin/eden; do @@ -49,14 +49,23 @@ for eden_bin in build*/bin/eden; do continue fi - # Extract build name - build_name=$(dirname "$eden_bin" | cut -d'/' -f1) + # Extract build path and make a unique folder name + build_path=$(dirname "$eden_bin") + build_name=$(echo "$build_path" | tr '/' '_') # Timestamp for unique log folder timestamp=$(date +%Y-%m-%d_%H-%M-%S) log_dir="$BASE_LOG_DIR/$build_name/$timestamp" mkdir -p "$log_dir" + # Save eden-cli version for this build + eden_cli="$build_path/eden-cli" + if [[ -x "$eden_cli" ]]; then + "$eden_cli" --version > "$log_dir/eden-cli-version.txt" 2>&1 + else + echo "eden-cli not found in $build_path, skipping version record" > "$log_dir/eden-cli-version.txt" + fi + echo "Running $eden_bin → logs will be saved in $log_dir" # Set MangoHud environment variables diff --git a/tools/test/compare_logs.py b/tools/test/compare_logs.py index 9c70fa992e..8352a8085c 100755 --- a/tools/test/compare_logs.py +++ b/tools/test/compare_logs.py @@ -8,6 +8,7 @@ import pandas as pd import matplotlib.pyplot as plt import glob import os +import re # Check required Python modules required_modules = ["pandas", "matplotlib"] @@ -49,19 +50,25 @@ colors = plt.colormaps['tab10'] stats = [] -# Track which folders have CSVs -folders_with_csv = set() - for i, csv_file in enumerate(csv_files): - folder = os.path.dirname(csv_file) - folders_with_csv.add(folder) + # Relative path from base folder + relative_path = os.path.relpath(csv_file, log_base_folder) + + # Read eden-cli version if exists + log_dir = os.path.dirname(csv_file) + version_file = os.path.join(log_dir, "eden-cli-version.txt") + if os.path.exists(version_file): + with open(version_file, "r") as f: + eden_version = f.read().strip() + else: + eden_version = "unknown version" # Corresponding summary file summary_file = csv_file.replace(".csv", "_summary.csv") # Skip empty CSVs if os.path.getsize(csv_file) == 0: - print(f"Skipping {csv_file}: file is empty") + print(f"Skipping {relative_path}: file is empty") continue # Read main CSV (skip system info lines) @@ -69,16 +76,22 @@ for i, csv_file in enumerate(csv_files): df.columns = df.columns.str.strip() if 'fps' not in df.columns: - print(f"Skipping {csv_file}: no 'fps' column found") + print(f"Skipping {relative_path}: no 'fps' column found") continue y = df['fps'] - x = range(len(y)) - # Compute statistics from main CSV - mean_fps = y.mean() - min_fps = y.min() - max_fps = y.max() + # --- FILTRAR OUTLIERS --- + mean_y = y.mean() + std_y = y.std() + y_filtered = y[(y >= mean_y - 3*std_y) & (y <= mean_y + 3*std_y)] + x_filtered = range(len(y_filtered)) + # ------------------------ + + # Compute statistics from filtered data + mean_fps = y_filtered.mean() + min_fps = y_filtered.min() + max_fps = y_filtered.max() # Read summary CSV if exists summary_text = "" @@ -93,15 +106,20 @@ for i, csv_file in enumerate(csv_files): except Exception as e: print(f"Could not read summary for {summary_file}: {e}") - stats.append((os.path.basename(csv_file), mean_fps, min_fps, max_fps, summary_text)) + # Store stats including version + stats.append((relative_path, mean_fps, min_fps, max_fps, summary_text, eden_version)) - # Plot FPS line with summary info - plt.plot(x, y, label=f"{os.path.basename(csv_file)} (avg={mean_fps:.1f}){summary_text}", color=colors(i % 10)) + # Plot FPS line with filtered data + plt.plot( + x_filtered, y_filtered, + label=f"{relative_path} (avg={mean_fps:.1f}) [{eden_version}]{summary_text}", + color=colors(i % 10) + ) # Configure plot plt.xlabel('Frame') plt.ylabel('FPS') -plt.title('FPS Comparison Across All Builds') +plt.title('FPS Comparison Across All Builds (Outliers Removed)') plt.legend() plt.grid(True) plt.tight_layout() @@ -111,10 +129,13 @@ png_file = os.path.join(os.getcwd(), "fps_comparison_all_builds.png") plt.savefig(png_file, dpi=200) plt.show() -# Print statistics in terminal -print("\nFPS Summary by file:") -for name, mean_fps, min_fps, max_fps, summary_text in stats: - print(f"{name}: mean={mean_fps:.1f}, min={min_fps:.1f}, max={max_fps:.1f}{summary_text}") +# Print concise statistics grouped by build folder +print("\nFPS Summary by build folder (concise):") +for name, mean_fps, min_fps, max_fps, summary_text, eden_version in stats: + log_folder = os.path.dirname(name) # pasta do CSV (_timestamp_) + build_folder = os.path.basename(os.path.dirname(log_folder)) # pasta do build* + version_short = eden_version.split()[0] if eden_version != "unknown version" else "unknown" + print(f"{build_folder} | {version_short} | mean={mean_fps:.1f}, min={min_fps:.1f}, max={max_fps:.1f}") print("\n----------------------------") print(f"Total CSV files processed: {len(stats)}")