import pyspark.pandas as ps # Add EWMA features to the DataFrame for the specified alpha values def add_ewma_features(df, alphas): for alpha in alphas: ewma_col_name = f"ewma_power_w{str(alpha).replace('.', '')}" windowSpec = Window.orderBy("Date") df[ewma_col_name] = df.Total_global_active_power.ewm(alpha=alpha).mean().round(2) return df alphas = [0.2, 0.8] # Convert into a pandas-on-Spark DataFrame, to use EWM function df2_pd = df2.pandas_api() df2_pd = add_ewma_features(df2_pd, alphas) # Convert back to a Spark DataFrame df2 = df2_pd.to_spark() df2.select("Date", "Total_global_active_power", "ewma_power_w02", "ewma_power_w08").sort("Date", ascending=False).show(5)