from bs4 import BeautifulSoup
import requests
import re

url = "https://en.wikipedia.org/wiki/List_of_Middle-earth_characters"
# Get the contents of the webpage in text format
data  = requests.get(url).text

soup = BeautifulSoup(data,"html.parser")

# Parse the HTML code to isolate rows containing character names
names = soup.find_all('li')

# Apply a second filter to isolate actual names
names_2 = []
for i in names:
    if """<li><a href="/wiki/""" in str(i):
        names_2.append(str(i))

# Retrieve final character names applying cleaning conditions
lotr_names = []
for i in names_2:
    candidate_name = re.findall(r'"(.*?)"', i)[1]
    conditions = 'Category:' not in candidate_name and 'Middle-earth ' not in candidate_name and 'Tolkien' not in candidate_name and 'Lord of the Rings' not in candidate_name and 'The Hobbit' not in candidate_name
    if conditions:
        lotr_names.append(candidate_name.replace(' (Middle-earth)',''))