Upload prepare_denmark_data.py with huggingface_hub

prepare_denmark_data.py

@@ -0,0 +1,95 @@
+"""
+Generate Denmark_data .mat files from the weather-denmark CSV
+to match the format expected by the EQL-Wind-Speed-Forecasting repo.
+"""
+import numpy as np
+import pandas as pd
+from scipy.io import savemat
+import os
+
+def main():
+    csv_path = '/tmp/weather-denmark/weather-denmark.csv'
+    df = pd.read_csv(csv_path)
+    df['DateTime'] = pd.to_datetime(df['DateTime'])
+
+    cities = ['Aalborg', 'Aarhus', 'Esbjerg', 'Odense', 'Roskilde']
+    features = ['Temp', 'Pressure', 'WindSpeed', 'WindDir']
+    target_feature = 'WindSpeed'
+    n_lags = 4
+    steps_ahead = 6
+    train_ratio = 0.9
+
+    df_hourly = []
+    for city in cities:
+        city_df = df[df['City'] == city].copy()
+        city_df = city_df.set_index('DateTime').sort_index()
+        city_hourly = city_df[features].resample('h').mean()
+        city_hourly = city_hourly.ffill().bfill()
+        city_hourly['City'] = city
+        df_hourly.append(city_hourly)
+
+    merged = pd.concat(df_hourly, axis=1, keys=cities)
+    merged = merged.dropna()
+    print(f"Total hourly samples after merge: {len(merged)}")
+
+    data_array = np.zeros((len(merged), len(cities), len(features)))
+    for i, city in enumerate(cities):
+        for j, feat in enumerate(features):
+            data_array[:, i, j] = merged[(city, feat)].values
+
+    valid_len = len(data_array) - steps_ahead - (n_lags - 1)
+    X = np.zeros((valid_len, len(cities), n_lags, len(features)))
+    Y = np.zeros((valid_len, len(cities)))
+
+    for t in range(valid_len):
+        start_idx = t + n_lags - 1
+        for lag in range(n_lags):
+            X[t, :, lag, :] = data_array[start_idx - lag, :, :]
+        Y[t, :] = data_array[start_idx + steps_ahead, :, features.index(target_feature)]
+
+    n_train = int(valid_len * train_ratio)
+    Xtr = X[:n_train]
+    Xtest = X[n_train:]
+    Ytr = Y[:n_train]
+    Ytest = Y[n_train:]
+
+    x_min = np.zeros((len(cities), len(features)))
+    x_max = np.zeros((len(cities), len(features)))
+    for i in range(len(cities)):
+        for j in range(len(features)):
+            vals = Xtr[:, i, :, j].flatten()
+            x_min[i, j] = vals.min()
+            x_max[i, j] = vals.max()
+            rng = x_max[i, j] - x_min[i, j]
+            if rng < 1e-8:
+                rng = 1.0
+            Xtr[:, i, :, j] = (Xtr[:, i, :, j] - x_min[i, j]) / rng
+            Xtest[:, i, :, j] = (Xtest[:, i, :, j] - x_min[i, j]) / rng
+
+    y_min_tr = np.zeros((1, len(cities)))
+    y_max_tr = np.zeros((1, len(cities)))
+    for i in range(len(cities)):
+        y_min_tr[0, i] = Ytr[:, i].min()
+        y_max_tr[0, i] = Ytr[:, i].max()
+        rng = y_max_tr[0, i] - y_min_tr[0, i]
+        if rng < 1e-8:
+            rng = 1.0
+        Ytr[:, i] = (Ytr[:, i] - y_min_tr[0, i]) / rng
+        Ytest[:, i] = (Ytest[:, i] - y_min_tr[0, i]) / rng
+
+    os.makedirs('Denmark_data/wind_speed', exist_ok=True)
+    os.makedirs('Denmark_data/temp', exist_ok=True)
+
+    for feat_dir, label in [('wind_speed', 'wind_speed'), ('temp', 'temp')]:
+        savemat(f'Denmark_data/{feat_dir}/step1.mat', {
+            'Xtr': Xtr, 'Xtest': Xtest, 'Ytr': Ytr, 'Ytest': Ytest,
+            'y_min_tr': y_min_tr, 'y_max_tr': y_max_tr,
+        }, do_compression=True)
+
+    np.savez('Denmark_data/scaling_params.npz',
+             x_min=x_min, x_max=x_max, y_min_tr=y_min_tr, y_max_tr=y_max_tr,
+             cities=cities, features=features)
+    print(f"Saved Denmark_data. Xtr:{Xtr.shape} Ytr:{Ytr.shape}")
+
+if __name__ == '__main__':
+    main()