CrbnsCat10n
02e488cd0d
new file: checkpoints_tree/task1_tr_tree/model_selection.csv new file: evaluation_outputs/task1_tr_tree/evaluation_train_all_samples.csv new file: evaluation_outputs/task1_tr_tree/evaluation_train_curve.png new file: evaluation_outputs/task1_tr_tree/evaluation_val_all_samples.csv new file: evaluation_outputs/task1_tr_tree/evaluation_val_curve.png new file: evaluation_outputs/task1_tr_tree/harmonic_5mm_1.55Hz_prediction.png new file: scripts_tree/__pycache__/config.cpython-310.pyc new file: scripts_tree/config.py new file: scripts_tree/evaluate.py new file: scripts_tree/predict_single.py new file: scripts_tree/train.py
178 lines
6.9 KiB
Python
178 lines
6.9 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import pickle
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
import matplotlib.pyplot as plt
|
|
import pandas as pd
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
if str(PROJECT_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
from scripts.dataset import build_datasets, report_to_text
|
|
|
|
try:
|
|
from .config import FEATURE_NAMES, evaluation_dir, make_experiment_config
|
|
except ImportError:
|
|
from config import FEATURE_NAMES, evaluation_dir, make_experiment_config
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Evaluate tree model on train/val/test splits.")
|
|
parser.add_argument("--split", choices=("train", "val", "test"), default="val")
|
|
parser.add_argument("--sample-index", type=int, default=0)
|
|
parser.add_argument("--all-samples", action="store_true")
|
|
parser.add_argument("--checkpoint", type=str, default=None)
|
|
return parser.parse_args()
|
|
|
|
|
|
def resolve_checkpoint_path(project_root: Path, checkpoint_arg: str | None) -> Path:
|
|
if checkpoint_arg:
|
|
return Path(checkpoint_arg).resolve()
|
|
return project_root / "checkpoints_tree" / "task1_tr_tree" / "best_tr_tree.pkl"
|
|
|
|
|
|
def relative_percent_error(true_value: float, pred_value: float) -> float:
|
|
return abs(pred_value - true_value) / max(abs(true_value), 1e-12) * 100.0
|
|
|
|
|
|
def evaluate_record(model, record) -> dict[str, float | str]:
|
|
feature_matrix = [record.features.tolist()]
|
|
pred_tr = max(float(model.predict(feature_matrix)[0]), 1e-6)
|
|
x_rms = float(record.x_rms.item())
|
|
y_rms = float(record.y_rms.item())
|
|
pred_rms = pred_tr * x_rms
|
|
row = {
|
|
"file_name": record.file_path.name,
|
|
"frequency_hz": float(record.frequency_hz),
|
|
"true_rms": y_rms,
|
|
"pred_rms": pred_rms,
|
|
"relative_error_percent": relative_percent_error(y_rms, pred_rms),
|
|
"x_rms": x_rms,
|
|
"pred_tr": pred_tr,
|
|
"true_tr": float(record.target_y_rms.item()),
|
|
}
|
|
for name, value in zip(FEATURE_NAMES, record.features.tolist()):
|
|
row[name] = float(value)
|
|
row["sampling_rate"] = float(record.sampling_rate)
|
|
return row
|
|
|
|
|
|
def save_all_samples_plot(result_df: pd.DataFrame, split: str, save_dir: Path) -> Path | None:
|
|
if result_df["frequency_hz"].isna().any():
|
|
return None
|
|
plot_df = result_df.sort_values("frequency_hz").reset_index(drop=True)
|
|
fig, axes = plt.subplots(2, 1, figsize=(12, 8))
|
|
fig.suptitle(f"Task1 TR Tree Evaluation | {split}")
|
|
|
|
axes[0].plot(plot_df["frequency_hz"], plot_df["true_rms"], marker="o", label="True RMS")
|
|
axes[0].plot(plot_df["frequency_hz"], plot_df["pred_rms"], marker="o", label="Pred RMS")
|
|
axes[0].set_xlabel("Frequency (Hz)")
|
|
axes[0].set_ylabel("RMS")
|
|
axes[0].grid(True, alpha=0.3)
|
|
axes[0].legend()
|
|
|
|
axes[1].bar(plot_df["frequency_hz"].astype(str), plot_df["relative_error_percent"], color="tab:orange")
|
|
axes[1].set_xlabel("Frequency (Hz)")
|
|
axes[1].set_ylabel("Relative Error (%)")
|
|
axes[1].grid(True, axis="y", alpha=0.3)
|
|
axes[1].tick_params(axis="x", labelrotation=45)
|
|
|
|
plt.tight_layout()
|
|
figure_path = save_dir / f"evaluation_{split}_curve.png"
|
|
plt.savefig(figure_path, dpi=180, bbox_inches="tight")
|
|
plt.close(fig)
|
|
return figure_path
|
|
|
|
|
|
def save_single_sample_plot(record, result: dict[str, float | str], split: str, save_dir: Path, sample_index: int) -> Path:
|
|
time_middle = record.time_middle.detach().cpu().numpy().reshape(-1)
|
|
x_middle = record.x_middle.detach().cpu().numpy().reshape(-1)
|
|
y_middle = record.y_middle.detach().cpu().numpy().reshape(-1)
|
|
frequency_hz = float(result["frequency_hz"])
|
|
pred_rms = float(result["pred_rms"])
|
|
true_rms = float(result["true_rms"])
|
|
|
|
fig, axes = plt.subplots(3, 1, figsize=(12, 10))
|
|
fig.suptitle(f"Task1 TR Tree | {split} | {result['file_name']}")
|
|
|
|
axes[0].plot(time_middle, x_middle, color="tab:blue")
|
|
axes[0].set_title("Input Base Excitation (Middle Segment)")
|
|
axes[0].set_xlabel("Time")
|
|
axes[0].set_ylabel("Acceleration")
|
|
axes[0].grid(True, alpha=0.3)
|
|
|
|
axes[1].plot(time_middle, y_middle, color="tab:green")
|
|
axes[1].set_title("True Top Response (Middle Segment)")
|
|
axes[1].set_xlabel("Time")
|
|
axes[1].set_ylabel("Acceleration")
|
|
axes[1].grid(True, alpha=0.3)
|
|
|
|
axes[2].bar(["True RMS", "Pred RMS"], [true_rms, pred_rms], color=["tab:green", "tab:orange"])
|
|
axes[2].set_title(
|
|
f"Freq: {frequency_hz:.4f} Hz | True RMS: {true_rms:.6f} | Pred RMS: {pred_rms:.6f} | "
|
|
f"Error: {float(result['relative_error_percent']):.2f}%"
|
|
)
|
|
axes[2].set_ylabel("RMS")
|
|
axes[2].grid(True, axis="y", alpha=0.3)
|
|
|
|
plt.tight_layout()
|
|
figure_path = save_dir / f"evaluation_{split}_s{sample_index}.png"
|
|
plt.savefig(figure_path, dpi=180, bbox_inches="tight")
|
|
plt.close(fig)
|
|
return figure_path
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
config = make_experiment_config()
|
|
ckpt_path = resolve_checkpoint_path(config.data.project_root, args.checkpoint)
|
|
with ckpt_path.open("rb") as handle:
|
|
bundle = pickle.load(handle)
|
|
model = bundle["model"]
|
|
|
|
_, raw_records, reports = build_datasets(config.data)
|
|
print(report_to_text(reports))
|
|
records = raw_records[args.split]
|
|
save_dir = evaluation_dir(config)
|
|
|
|
if args.all_samples:
|
|
rows = [evaluate_record(model, record) for record in records]
|
|
result_df = pd.DataFrame(rows).sort_values(["relative_error_percent", "file_name"]).reset_index(drop=True)
|
|
csv_path = save_dir / f"evaluation_{args.split}_all_samples.csv"
|
|
result_df.to_csv(csv_path, index=False)
|
|
fig_path = save_all_samples_plot(result_df, args.split, save_dir)
|
|
summary = {
|
|
"count": len(result_df),
|
|
"mean_error_percent": float(result_df["relative_error_percent"].mean()),
|
|
"median_error_percent": float(result_df["relative_error_percent"].median()),
|
|
"max_error_percent": float(result_df["relative_error_percent"].max()),
|
|
"min_error_percent": float(result_df["relative_error_percent"].min()),
|
|
}
|
|
print(f"Checkpoint: {ckpt_path}")
|
|
print(f"Model: {bundle['model_name']}")
|
|
print(f"Summary: {summary}")
|
|
print(f"CSV saved to: {csv_path}")
|
|
if fig_path is not None:
|
|
print(f"Figure saved to: {fig_path}")
|
|
print(result_df.to_string(index=False))
|
|
return
|
|
|
|
record = records[args.sample_index]
|
|
result = evaluate_record(model, record)
|
|
fig_path = save_single_sample_plot(record, result, args.split, save_dir, args.sample_index)
|
|
print(f"Checkpoint: {ckpt_path}")
|
|
print(f"Model: {bundle['model_name']}")
|
|
print(f"Sample file: {result['file_name']}")
|
|
print(f"True RMS: {float(result['true_rms']):.6f}")
|
|
print(f"Pred RMS: {float(result['pred_rms']):.6f}")
|
|
print(f"Relative RMS Error (%): {float(result['relative_error_percent']):.4f}")
|
|
print(f"Figure saved to: {fig_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|