I'm currently trying to play around with selenium and click this load more button every time it comes up. but it works up until near the end and it stops. Can anyone suggest what i my be doing wrong? Sorry for the sloppy code I'm a bit new to python.
import json, time
from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
class Scrape():
def __init__(self):
all_products = []
driver = Chrome()
url = "https://shop.bodybuilding.com/collections/protein?sort_by=relevency&page_number=1"
driver.maximize_window()
driver.get(url)
driver.implicitly_wait(10)
morebtn = driver.find_element(by=By.CLASS_NAME, value="InstantSearch__load-more")
while(morebtn):
try:
morebtn.click()
except Exception:
print(Exception())
products = driver.find_elements(by=By.CLASS_NAME, value="Anchor")
for product in products:
if(product.text != ""):
product_dict = {}
product_dict['name'] = product.text
all_products.append(product_dict)
driver.quit()
f = open('bodybuilding.json', 'w')
f.write(json.dumps(all_products, indent=2))
f.close()
test = Scrape()