from sklearn.preprocessing import MultiLabelBinarizer popular_genres = ['Comedy', 'Action', 'Fantasy', 'Adventure', 'Kids', 'Drama', 'Sci-Fi', 'Music', 'Shounen', 'Slice of Life'] def create_genre_flags(df, popular_genres): df = df.dropna(subset=['Genres']) df['Genres'] = df['Genres'].apply(lambda x:",".join(s.strip() for s in x.split(","))) # use MultiLabelBinarizer to create a one-hot encoded dataframe of the genres mlb = MultiLabelBinarizer() genre_df = pd.DataFrame(mlb.fit_transform(df['Genres'].str.split(',')), columns=mlb.classes_, index=df.index) # create a new dataframe with the movie id and genre columns new_df = pd.concat([df['anime_id'], genre_df[popular_genres]], axis=1) new_df.columns = ['anime_id'] + popular_genres return new_df anime_genre_info_df = create_genre_flags(anime_info_df,popular_genres) anime_info_df_final = anime_info_df.merge(anime_genre_info_df,on='anime_id') anime_info_df_final.columns = [col if col=='anime_id' else f"ANIME_FEATURE {col}".upper() for col in anime_info_df_final.columns] user_info.columns = [col if col=='user_id' else f"USER_FEATURE {col}".upper() for col in user_info.columns] train_interim = relavence_scores.merge(anime_info_df_final) train = train_interim.merge(user_info,how='inner')