# Add window average fields to the DataFrame for the specified window sizes def add_window_avg_features(df, window_sizes): for window_size in window_sizes: window_col_name = f"avg_power_l{window_size}" windowSpec = Window.orderBy("Date").rowsBetween(-window_size, 0) df = df.withColumn(window_col_name, round(avg(col("Total_global_active_power")).over(windowSpec), 2)) return df window_sizes = [14, 30] df2 = add_window_avg_features(df2, window_sizes) df2.select("Date", "Total_global_active_power", "avg_power_l14", "avg_power_l30").sort("Date", ascending=False).show(5)