Changes to be committed:

modified:   .gitignore
	deleted:    Figure_1.png
	deleted:    checkpoints/forward/best_tcn_model.pt
	deleted:    checkpoints/forward/training_history.csv
	deleted:    checkpoints_mlp/task1_feature_mlp/best_feature_mlp.pt
	deleted:    checkpoints_mlp/task1_feature_mlp/training_history.csv
	deleted:    checkpoints_rms/forward_rms/best_rms_model.pt
	deleted:    checkpoints_rms/forward_rms/training_history.csv
	deleted:    checkpoints_tree/task1_tr_tree/best_tr_tree.pkl
	deleted:    checkpoints_tree/task1_tr_tree/model_selection.csv
	deleted:    evaluation_outputs/forward/evaluation_forward_test_b0_s0.png
	deleted:    evaluation_outputs/forward/evaluation_forward_val_b0_s0.png
	deleted:    evaluation_outputs/forward/evaluation_forward_val_b3_s0.png
	deleted:    evaluation_outputs/forward_rms/evaluation_test_all_samples.csv
	deleted:    evaluation_outputs/forward_rms/evaluation_test_s0.png
	deleted:    evaluation_outputs/forward_rms/evaluation_train_all_samples.csv
	deleted:    evaluation_outputs/forward_rms/evaluation_val_all_samples.csv
	deleted:    evaluation_outputs/forward_rms/evaluation_val_s0.png
	deleted:    evaluation_outputs/forward_rms/evaluation_val_s0_waveform.png
	deleted:    evaluation_outputs/task1_feature_mlp/evaluation_train_all_samples.csv
	deleted:    evaluation_outputs/task1_feature_mlp/evaluation_train_curve.png
	deleted:    evaluation_outputs/task1_feature_mlp/evaluation_val_all_samples.csv
	deleted:    evaluation_outputs/task1_feature_mlp/evaluation_val_curve.png
	deleted:    evaluation_outputs/task1_feature_mlp/evaluation_val_s0.png
	deleted:    evaluation_outputs/task1_feature_mlp/harmonic_5mm_0.75Hz_prediction.png
	deleted:    evaluation_outputs/task1_feature_mlp/harmonic_5mm_1.55Hz_prediction.png
	deleted:    evaluation_outputs/task1_tr_tree/evaluation_train_all_samples.csv
	deleted:    evaluation_outputs/task1_tr_tree/evaluation_train_curve.png
	deleted:    evaluation_outputs/task1_tr_tree/evaluation_val_all_samples.csv
	deleted:    evaluation_outputs/task1_tr_tree/evaluation_val_curve.png
	deleted:    evaluation_outputs/task1_tr_tree/harmonic_5mm_1.55Hz_prediction.png
	deleted:    sanity_check_alignment_forward.png
	new file:   scripts/README.md
	modified:   scripts/__pycache__/config.cpython-310.pyc
	modified:   scripts/__pycache__/dataset.cpython-310.pyc
	deleted:    scripts/__pycache__/model.cpython-310.pyc
	modified:   scripts/config.py
	modified:   scripts/dataset.py
	modified:   scripts/evaluate.py
	deleted:    scripts/model.py
	modified:   scripts/predict_single.py
	deleted:    scripts/train.py
	new file:   scripts/train_final.py
	deleted:    scripts_tree/__pycache__/config.cpython-310.pyc
	deleted:    scripts_tree/config.py
	deleted:    scripts_tree/evaluate.py
	deleted:    scripts_tree/predict_single.py
	deleted:    scripts_tree/train.py
	deleted:    src/__pycache__/config.cpython-310.pyc
	deleted:    src/__pycache__/config.cpython-314.pyc
	deleted:    src/__pycache__/dataset.cpython-310.pyc
	deleted:    src/__pycache__/dataset.cpython-314.pyc
	deleted:    src/__pycache__/model.cpython-310.pyc
	deleted:    src/__pycache__/model.cpython-314.pyc
	deleted:    src/config.py
	deleted:    src/dataset.py
	deleted:    src/evaluate.py
	deleted:    src/model.py
	deleted:    src/sanity_check.py
	deleted:    src/train.py
	deleted:    src_new/__pycache__/config.cpython-310.pyc
	deleted:    src_new/__pycache__/dataset.cpython-310.pyc
	deleted:    src_new/__pycache__/evaluate.cpython-310.pyc
	deleted:    src_new/__pycache__/model.cpython-310.pyc
	deleted:    src_new/__pycache__/train.cpython-310.pyc
	deleted:    src_new/config.py
	deleted:    src_new/dataset.py
	deleted:    src_new/evaluate.py
	deleted:    src_new/model.py
	deleted:    src_new/train.py
	deleted:    src_old/__init__.py
	deleted:    src_old/__pycache__/config.cpython-310.pyc
	deleted:    src_old/__pycache__/config.cpython-314.pyc
	deleted:    src_old/__pycache__/dataset.cpython-310.pyc
	deleted:    src_old/__pycache__/dataset.cpython-314.pyc
	deleted:    src_old/__pycache__/evaluate.cpython-314.pyc
	deleted:    src_old/__pycache__/model.cpython-310.pyc
	deleted:    src_old/__pycache__/model.cpython-314.pyc
	deleted:    src_old/__pycache__/train.cpython-310.pyc
	deleted:    src_old/__pycache__/train.cpython-314.pyc
	deleted:    src_old/config.py
	deleted:    src_old/dataset.py
	deleted:    src_old/evaluate.py
	deleted:    src_old/model.py
	deleted:    src_old/train.py
This commit is contained in:
2026-05-06 14:37:37 +08:00
parent 02e488cd0d
commit fc8fcc2746
85 changed files with 456 additions and 5067 deletions

View File

@@ -3,12 +3,9 @@ from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
import re
from typing import Any
import numpy as np
import pandas as pd
import torch
from torch.utils.data import DataLoader, Dataset
try:
from .config import CORE_FEATURE_NAMES, DataConfig, ExperimentConfig
@@ -85,7 +82,6 @@ def get_middle_segment(
peak_cv = float(split_peaks.std() / max(split_peaks.mean(), config.normalization_eps))
window_rms = calculate_rms(window_y)
# Prefer windows that are stable inside the middle candidate region while keeping enough energy.
score = rms_cv + 0.35 * peak_cv - 0.05 * (window_rms / max(candidate_rms, config.normalization_eps))
if score < best_score:
best_score = score
@@ -134,106 +130,24 @@ def get_dominant_frequency(signal: np.ndarray, sampling_rate: float) -> float:
return _parabolic_peak_frequency(freqs, magnitudes, peak_index)
def compute_harmonic_fit_features(signal: np.ndarray, sampling_rate: float, frequency_hz: float) -> tuple[float, float]:
signal = np.asarray(signal, dtype=np.float64).reshape(-1)
if signal.size < 4 or frequency_hz <= 0.0:
return 0.0, 1.0
time_axis = np.arange(signal.size, dtype=np.float64) / max(sampling_rate, 1e-12)
omega_t = 2.0 * np.pi * frequency_hz * time_axis
design = np.stack([np.sin(omega_t), np.cos(omega_t), np.ones_like(omega_t)], axis=1)
coefficients, _, _, _ = np.linalg.lstsq(design, signal, rcond=None)
fitted = design @ coefficients
harmonic_amplitude = float(np.sqrt(coefficients[0] ** 2 + coefficients[1] ** 2))
residual = signal - fitted
residual_ratio = calculate_rms(residual) / max(calculate_rms(signal), 1e-6)
return harmonic_amplitude, float(residual_ratio)
def compute_spectral_features(signal: np.ndarray, sampling_rate: float) -> tuple[float, float, float, float, float]:
freqs, amplitudes = _compute_windowed_spectrum(signal, sampling_rate)
if amplitudes.size == 0:
return 0.0, 0.0, 0.0, 0.0, 0.0
powers = np.square(amplitudes)
amplitudes[0] = 0.0
powers[0] = 0.0
band_mask = (freqs >= 0.1) & (freqs <= 5.0)
if not np.any(band_mask):
return 0.0, 0.0, 0.0, 0.0, 0.0
band_amplitudes = np.where(band_mask, amplitudes, 0.0)
dominant_index = int(np.argmax(band_amplitudes))
dominant_amplitude = float(amplitudes[dominant_index])
dominant_freq = _parabolic_peak_frequency(freqs, amplitudes, dominant_index)
local_mask = np.abs(freqs - dominant_freq) <= 0.10
dominant_energy = float(powers[local_mask].sum())
total_band_energy = float(powers[band_mask].sum())
dominant_energy_ratio = dominant_energy / max(total_band_energy, 1e-12)
spectral_centroid = float((freqs[band_mask] * powers[band_mask]).sum() / max(total_band_energy, 1e-12))
background_mask = band_mask & (~local_mask)
background_level = float(np.median(amplitudes[background_mask])) if np.any(background_mask) else 0.0
spectral_peak_prominence = dominant_amplitude / max(background_level, 1e-6)
half_power_level = dominant_amplitude / np.sqrt(2.0)
left_index = dominant_index
right_index = dominant_index
while left_index > 0 and amplitudes[left_index] >= half_power_level:
left_index -= 1
while right_index < amplitudes.size - 1 and amplitudes[right_index] >= half_power_level:
right_index += 1
half_power_bandwidth = float(freqs[right_index] - freqs[left_index]) if right_index > left_index else 0.0
return dominant_amplitude, dominant_energy_ratio, spectral_centroid, spectral_peak_prominence, half_power_bandwidth
def theoretical_acceleration(frequency_hz: float, amplitude_m: float) -> float:
return float((2.0 * np.pi * frequency_hz) ** 2 * amplitude_m)
def extract_core_features(
signal: np.ndarray,
sampling_rate: float,
original_length: int,
config: DataConfig,
known_frequency_hz: float | None = None,
) -> np.ndarray:
frequency_hz = known_frequency_hz if known_frequency_hz is not None else get_dominant_frequency(signal, sampling_rate)
x_rms = calculate_rms(signal)
x_peak = float(np.max(np.abs(signal))) if signal.size > 0 else 0.0
x_peak_to_peak = float(np.max(signal) - np.min(signal)) if signal.size > 0 else 0.0
crest_factor = x_peak / max(x_rms, config.normalization_eps)
dominant_frequency = get_dominant_frequency(signal, sampling_rate)
frequency_squared = dominant_frequency * dominant_frequency
inverse_frequency = 1.0 / max(dominant_frequency, 1e-6)
log_frequency = float(np.log(max(dominant_frequency, 1e-6)))
middle_length_ratio = float(signal.size) / float(max(original_length, 1))
(
dominant_amplitude,
dominant_energy_ratio,
spectral_centroid,
spectral_peak_prominence,
half_power_bandwidth,
) = compute_spectral_features(signal, sampling_rate)
harmonic_fit_amplitude, harmonic_fit_residual_ratio = compute_harmonic_fit_features(
signal,
sampling_rate,
dominant_frequency,
)
signal_mean = float(np.mean(signal)) if signal.size > 0 else 0.0
return np.asarray(
[
dominant_frequency,
frequency_squared,
inverse_frequency,
log_frequency,
frequency_hz,
frequency_hz * frequency_hz,
x_rms,
x_peak,
x_peak_to_peak,
crest_factor,
middle_length_ratio,
dominant_amplitude,
dominant_energy_ratio,
harmonic_fit_amplitude,
harmonic_fit_residual_ratio,
spectral_peak_prominence,
half_power_bandwidth,
spectral_centroid,
signal_mean,
theoretical_acceleration(frequency_hz, config.harmonic_amplitude_m),
],
dtype=np.float32,
)
@@ -277,29 +191,26 @@ def load_aligned_signals(file_path: Path, config: DataConfig) -> tuple[np.ndarra
@dataclass
class SampleRecord:
file_path: Path
split: str
frequency_hz: float
features: torch.Tensor
x_rms: torch.Tensor
y_rms: torch.Tensor
target_y_rms: torch.Tensor
target_log_y_rms: torch.Tensor
time_middle: torch.Tensor
x_middle: torch.Tensor
y_middle: torch.Tensor
features: np.ndarray
x_rms: float
y_rms: float
target_tr: float
time_middle: np.ndarray
x_middle: np.ndarray
y_middle: np.ndarray
sampling_rate: float
interpolation_count: int = 0
@dataclass
class SplitLoadReport:
split: str
class LoadReport:
loaded_files: list[str] = field(default_factory=list)
skipped_files: list[tuple[str, str]] = field(default_factory=list)
interpolated_files: dict[str, int] = field(default_factory=dict)
def to_lines(self) -> list[str]:
lines = [f"[{self.split}] loaded={len(self.loaded_files)} skipped={len(self.skipped_files)}"]
lines = [f"[all_data] loaded={len(self.loaded_files)} skipped={len(self.skipped_files)}"]
for file_name, reason in self.skipped_files:
lines.append(f" - skipped {file_name}: {reason}")
for file_name, count in self.interpolated_files.items():
@@ -308,45 +219,12 @@ class SplitLoadReport:
return lines
@dataclass
class NormalizationStats:
feature_mean: torch.Tensor
feature_std: torch.Tensor
target_mean: torch.Tensor
target_std: torch.Tensor
def list_harmonic_files(config: DataConfig) -> list[Path]:
files = sorted(config.data_root.rglob(config.harmonic_pattern))
return [file_path for file_path in files if extract_frequency_hz(file_path.name) is not None]
class FeatureDataset(Dataset):
def __init__(self, records: list[SampleRecord], normalization: NormalizationStats) -> None:
self.records = records
self.normalization = normalization
def __len__(self) -> int:
return len(self.records)
def __getitem__(self, index: int) -> dict[str, Any]:
record = self.records[index]
feature_norm = (record.features - self.normalization.feature_mean) / self.normalization.feature_std
target_norm = (record.target_y_rms - self.normalization.target_mean) / self.normalization.target_std
return {
"x": feature_norm,
"target": target_norm,
"target_raw": record.target_y_rms,
"target_log_raw": record.target_log_y_rms,
"x_rms_raw": record.x_rms,
"y_rms_raw": record.y_rms,
"frequency_hz": torch.tensor(record.frequency_hz, dtype=torch.float32),
"features_raw": record.features,
"file_name": record.file_path.name,
}
def list_split_files(config: DataConfig, split: str) -> list[Path]:
split_dir = getattr(config, f"{split}_dir")
return sorted(split_dir.glob(config.csv_pattern))
def build_record_from_file(file_path: Path, split: str, config: DataConfig) -> SampleRecord:
def build_record_from_file(file_path: Path, config: DataConfig) -> SampleRecord:
time_values, x_values, y_values, interpolation_count = load_aligned_signals(file_path, config)
time_middle, x_middle, y_middle = get_middle_segment(time_values, x_values, y_values, config)
sampling_rate = estimate_sampling_rate(time_middle)
@@ -356,34 +234,32 @@ def build_record_from_file(file_path: Path, split: str, config: DataConfig) -> S
x_rms = calculate_rms(x_middle)
y_rms = calculate_rms(y_middle)
transmission_ratio = float(y_rms / max(x_rms, config.normalization_eps))
target_y_rms = transmission_ratio
target_log_y_rms = float(np.log(max(transmission_ratio, config.normalization_eps)))
features = extract_core_features(x_middle, sampling_rate, len(x_values), config)
target_tr = float(y_rms / max(x_rms, config.normalization_eps))
features = extract_core_features(x_middle, sampling_rate, config, known_frequency_hz=frequency_hz)
return SampleRecord(
file_path=file_path,
split=split,
frequency_hz=frequency_hz,
features=torch.tensor(features, dtype=torch.float32),
x_rms=torch.tensor([x_rms], dtype=torch.float32),
y_rms=torch.tensor([y_rms], dtype=torch.float32),
target_y_rms=torch.tensor([target_y_rms], dtype=torch.float32),
target_log_y_rms=torch.tensor([target_log_y_rms], dtype=torch.float32),
time_middle=torch.tensor(time_middle, dtype=torch.float64),
x_middle=torch.tensor(x_middle[:, None], dtype=torch.float32),
y_middle=torch.tensor(y_middle[:, None], dtype=torch.float32),
sampling_rate=sampling_rate,
frequency_hz=float(frequency_hz),
features=features,
x_rms=float(x_rms),
y_rms=float(y_rms),
target_tr=target_tr,
time_middle=np.asarray(time_middle, dtype=np.float64),
x_middle=np.asarray(x_middle, dtype=np.float32).reshape(-1),
y_middle=np.asarray(y_middle, dtype=np.float32).reshape(-1),
sampling_rate=float(sampling_rate),
interpolation_count=interpolation_count,
)
def load_split_records(config: DataConfig, split: str) -> tuple[list[SampleRecord], SplitLoadReport]:
def load_all_records(config: ExperimentConfig | DataConfig) -> tuple[list[SampleRecord], LoadReport]:
data_config = config.data if isinstance(config, ExperimentConfig) else config
records: list[SampleRecord] = []
report = SplitLoadReport(split=split)
for file_path in list_split_files(config, split):
report = LoadReport()
for file_path in list_harmonic_files(data_config):
try:
record = build_record_from_file(file_path, split, config)
record = build_record_from_file(file_path, data_config)
except ValueError as error:
report.skipped_files.append((file_path.name, str(error)))
continue
@@ -393,86 +269,26 @@ def load_split_records(config: DataConfig, split: str) -> tuple[list[SampleRecor
report.interpolated_files[file_path.name] = record.interpolation_count
if not records:
raise RuntimeError(f"No usable records found for split='{split}'.")
raise RuntimeError("No usable harmonic records found in data_root.")
return records, report
def fit_normalization(records: list[SampleRecord], config: DataConfig) -> NormalizationStats:
feature_all = torch.stack([record.features for record in records], dim=0)
target_all = torch.cat([record.target_y_rms for record in records], dim=0)
feature_std = torch.clamp(feature_all.std(dim=0, unbiased=False), min=config.normalization_eps)
target_std = torch.clamp(target_all.std(dim=0, unbiased=False).view(1), min=config.normalization_eps)
return NormalizationStats(
feature_mean=feature_all.mean(dim=0),
feature_std=feature_std,
target_mean=target_all.mean(dim=0, keepdim=True),
target_std=target_std,
)
def records_to_frame(records: list[SampleRecord]) -> pd.DataFrame:
rows = []
for record in records:
row = {
"file_name": record.file_path.name,
"frequency_hz": record.frequency_hz,
"x_rms": record.x_rms,
"y_rms": record.y_rms,
"target_tr": record.target_tr,
"sampling_rate": record.sampling_rate,
}
for feature_name, feature_value in zip(CORE_FEATURE_NAMES, record.features.tolist()):
row[feature_name] = float(feature_value)
rows.append(row)
return pd.DataFrame(rows)
def build_datasets(
config: ExperimentConfig | DataConfig,
) -> tuple[dict[str, FeatureDataset], dict[str, list[SampleRecord]], dict[str, SplitLoadReport]]:
data_config = config.data if isinstance(config, ExperimentConfig) else config
train_records, train_report = load_split_records(data_config, "train")
normalization = fit_normalization(train_records, data_config)
val_records, val_report = load_split_records(data_config, "val")
test_records, test_report = load_split_records(data_config, "test")
raw_records = {
"train": train_records,
"val": val_records,
"test": test_records,
}
datasets = {
split: FeatureDataset(records, normalization)
for split, records in raw_records.items()
}
reports = {
"train": train_report,
"val": val_report,
"test": test_report,
}
return datasets, raw_records, reports
def build_dataloaders(
config: ExperimentConfig | DataConfig,
) -> tuple[dict[str, DataLoader], dict[str, FeatureDataset], dict[str, list[SampleRecord]], dict[str, SplitLoadReport]]:
experiment_config = config if isinstance(config, ExperimentConfig) else ExperimentConfig(data=config)
datasets, raw_records, reports = build_datasets(experiment_config)
batch_size = experiment_config.train.batch_size
loaders = {
"train": DataLoader(datasets["train"], batch_size=batch_size, shuffle=True),
"val": DataLoader(datasets["val"], batch_size=batch_size, shuffle=False),
"test": DataLoader(datasets["test"], batch_size=batch_size, shuffle=False),
}
return loaders, datasets, raw_records, reports
def report_to_text(reports: dict[str, SplitLoadReport]) -> str:
lines: list[str] = []
for split in ("train", "val", "test"):
lines.extend(reports[split].to_lines())
return "\n".join(lines)
def checkpoint_normalization_payload(normalization: NormalizationStats) -> dict[str, list[float]]:
return {
"feature_mean": normalization.feature_mean.tolist(),
"feature_std": normalization.feature_std.tolist(),
"target_mean": normalization.target_mean.tolist(),
"target_std": normalization.target_std.tolist(),
"feature_names": list(CORE_FEATURE_NAMES),
}
def normalization_from_payload(payload: dict[str, Any]) -> NormalizationStats:
return NormalizationStats(
feature_mean=torch.tensor(payload["feature_mean"], dtype=torch.float32),
feature_std=torch.tensor(payload["feature_std"], dtype=torch.float32),
target_mean=torch.tensor(payload["target_mean"], dtype=torch.float32),
target_std=torch.tensor(payload["target_std"], dtype=torch.float32),
)
def report_to_text(report: LoadReport) -> str:
return "\n".join(report.to_lines())