Forums

def data_place_crawler():


  chrome_options = webdriver.ChromeOptions()
  chrome_options.add_argument("--no-sandbox")
  chrome_options.add_argument("--headless")
  chrome_options.add_argument("--disable-gpu")
  driver = webdriver.Chrome(options=chrome_options) #line 23 - 24 for hosting test

  try:
      dfPlace = []


      #time.sleep(100)
      foodID_list =[]
      businessName_list =[]
      url_list =[]

      food_list = (pd.read_csv("place_url.csv")['foodid'])
      placeURL_list = (pd.read_csv("place_url.csv")['placeURL'])
      foodPlace_dict = dict(zip(food_list, placeURL_list))

      for food, url in foodPlace_dict.items():
          driver.get(url)
          time.sleep(3)

          #Find scroll layout
          scrollable_div = driver.find_element(By.XPATH, '//* 
                                             [@id="QA0Szd"]/div/div/div[1]/div[2]/div/div[1]/div/div/div[2]/div[1]')
          #Scroll as many times as necessary to load all reviews
          for i in range(0,2):
                  driver.execute_script('arguments[0].scrollTop = arguments[0].scrollHeight',
                          scrollable_div)
                  time.sleep(3)


          businessDetail = driver.find_elements(By.CLASS_NAME, "hfpxzc")

          for lnk in businessDetail:
              foodID_list.append(food)
              businessName_list.append(lnk.get_attribute('aria-label'))
              url_list.append(lnk.get_attribute('href'))

      df = pd.DataFrame(list(zip(foodID_list,businessName_list, url_list)),
                      columns=['Food ID','Business Name', 'URL Link'])

      print(df)
      df.to_csv('/home/smtan041/FYP/google_place.csv',index=False)

      dfPlace = df.values.tolist()
      messageComplete = "Completed!"

      return dfPlace , messageComplete
  finally:
      driver.quit()

 Incompatible release of chromedriver (version 90.0.4430.24) detected in PATH: /usr/local/bin/chromedriver

15:37 ~/FYP (master)$ pip show selenium
Name: selenium
Version: 4.7.2
Summary: 
Home-page: https://www.selenium.dev
Author: 
Author-email: 
License: Apache 2.0
Location: /home/smtan041/.local/lib/python3.10/site-packages
Requires: certifi, trio, trio-websocket, urllib3
Required-by: 
15:38 ~/FYP (master)$