-
Notifications
You must be signed in to change notification settings - Fork 1
/
youtube_webdriver.py
67 lines (54 loc) · 2.1 KB
/
youtube_webdriver.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
"""
Webdriver script to get youtube urls and navigate through "Up next"
"""
import sys
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import StaleElementReferenceException, NoSuchElementException
def youtube_navigate(start_url, total=10, delay=5):
"Navigate through youtube with 'Up next'"
# xPath = "//div[@class='autoplay-bar']/div[@class='watch-sidebar-body']"
xPath = "//a[@class='yt-simple-endpoint style-scope ytd-compact-video-renderer']"
# xPath = "//ul[@class='video-list']"
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--mute-audio")
driver = webdriver.Chrome(chrome_options=chrome_options)
# driver = webdriver.PhantomJS()
driver.implicitly_wait(delay)
youtube_urls = []
try:
driver.get(start_url)
for _ in range(total):
# youtube_urls.append((driver.title, driver.current_url))
current_url = driver.current_url
print(current_url)
youtube_urls.append(current_url)
element = driver.find_element_by_xpath(xPath)
try:
element.click()
except StaleElementReferenceException:
element = driver.find_element_by_xpath(xPath)
element.click()
# driver.save_screenshot("test" + str(i) + ".png")
time.sleep(1)
except NoSuchElementException:
print("No such element, maybe the page is not loaded")
except:
print("Unexpected error:", sys.exc_info()[0])
driver.close()
return youtube_urls
def wait(driver, id, delay=10):
"Wait until element id appears"
try:
element = WebDriverWait(driver, delay).until(
EC.presence_of_element_located((By.ID, id))
)
return element
except TimeoutException:
print("Loading took too much time!")
driver.close()
raise