I was practicing trying to get links to videos from a youtube channel using Selenium. And it it returns None
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
import time
s = Service('C:\\Users\\user\\Desktop\\chromedriver-win64\\chromedriver.exe')
link = 'https://www.youtube.com/@PythonToday/videos'
browser = webdriver.Chrome(service=s)
browser.maximize_window()
browser.implicitly_wait(40)
browser.get(link)
time.sleep(10)
all_elements = browser.find_elements(By.ID, 'dismissible')
for item in all_elements:
jj = item.find_element(By.ID, 'thumbnail')
vv = jj.get_attribute('href')
print(vv)
It should return links to videos on the channel, but instead it returns all None. However if I use finding elements using BeautifulSoup instead of finding elements in selenium, as shown below, it returns the links and works as it should. in both cases I use the same Id's for finding elements, and in the first case it doesn't work, but in the second one it works. Why isn't this working properly with selenium?
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from bs4 import BeautifulSoup
import time
s = Service('C:\\Users\\user\\Desktop\\chromedriver-win64\\chromedriver.exe')
link = 'https://www.youtube.com/@PythonToday/videos'
browser = webdriver.Chrome(service=s)
browser.maximize_window()
browser.get(link)
time.sleep(10)
soup = BeautifulSoup(browser.page_source, 'html.parser')
all_elements = soup.find_all('div', id='dismissible')
for item in all_elements:
jj = item.find('a', id='thumbnail')
vv = jj.get('href')
print(vv)
Also I've tried to change the last part of the not working code adding Expected Conditions and WebDriverWait and tried to look for the elements with XPATH as shown below. But it still returns all None
all_elements = WebDriverWait(browser, 10).until(EC.presence_of_all_elements_located((By.XPATH, '//*[@id="dismissible"]')))
for item in all_elements:
jj = WebDriverWait(item, 10).until(EC.presence_of_element_located((By.XPATH, '//*[@id="thumbnail"]')))
vv = jj.get_attribute('href')
print(vv)