# create stopwords list standard_stopwords = list(stopwords.words('english')) # extended list of English stopwords stopwords_extended = [ "0o", ..] # additional tokens to remove additional_stopwords = ['blue','buffalo','dog','food','ha','month','ago'] # combine standard, extended stopwords, and additional tokens full_stopwords = standard_stopwords + additional_stopwords + stopwords_extended # define tokenizer retrurning lemmatized text without numbers class LemmaTokenizer: def __init__(self): self.wnl = WordNetLemmatizer() def __call__(self, doc): doc = re.sub(r'd+', '', doc) # clean numbers return [self.wnl.lemmatize(t) for t in word_tokenize(doc)] # lemmatize # vectorizer makes data processing and generates bigrams vectorizer_model = CountVectorizer(tokenizer=LemmaTokenizer(), ngram_range=(2, 2), stop_words=full_stopwords) # set-up model model = BERTopic(n_gram_range=(2,2), # returns bigrams nr_topics=9, # generate 9 topics, leave -1 for outliers top_n_words=20, # return top 20 bigrams min_topic_size=20, # topics contains at least 20 tokens vectorizer_model=vectorizer_model, umap_model = UMAP(random_state=1)) # setting seed topics reproduce # fit model to data topics, probabilities = model.fit_transform(docs)