Forums

import json, time
from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By

class Scrape():

    def __init__(self):
        all_products = []

        driver = Chrome()

        url = "https://shop.bodybuilding.com/collections/protein?sort_by=relevency&page_number=1"
        driver.maximize_window()
        driver.get(url)

        driver.implicitly_wait(10)

        morebtn = driver.find_element(by=By.CLASS_NAME, value="InstantSearch__load-more")

        while(morebtn):
            try:
                morebtn.click()
            except Exception:
                print(Exception())

        products = driver.find_elements(by=By.CLASS_NAME, value="Anchor")

        for product in products:
            if(product.text != ""):
                product_dict = {}
                product_dict['name'] = product.text

                all_products.append(product_dict)
        driver.quit()

        f = open('bodybuilding.json', 'w')
        f.write(json.dumps(all_products, indent=2))
        f.close()


test = Scrape()