유튜브 크롤링

import requests
from bs4 import BeautifulSoup
import time
import urllib.request 
import re
import pandas as pd
from selenium.webdriver import Chrome
from selenium.webdriver.common.keys import Keys
import datetime as dt

browser =  Chrome(executable_path='E:/chromedriver_win32_1/chromedriver.exe')

base_url = 'http://www.youtube.com'
browser.get(base_url)
browser.maximize_window()

# copy full xpath를 했습니다
# 추후에 xpath,css selector를 더 공부하여 손쉬운 접근법으로 개선하겠습니다.

browser.find_elements_by_xpath('/html/body/ytd-app/div/div/ytd-masthead/div[3]/div[2]/ytd-searchbox/form/div/div[1]/div/div[2]/input')[0].click()

browser.find_elements_by_xpath('/html/body/ytd-app/div/div/ytd-masthead/div[3]/div[2]/ytd-searchbox/form/div/div[1]/div/div[2]/input')[0].send_keys('dingo freestyle') #원하는 유튜버 입력
browser.find_elements_by_xpath('/html/body/ytd-app/div/div/ytd-masthead/div[3]/div[2]/ytd-searchbox/form/div/div[1]/div/div[2]/input')[0].send_keys(Keys.RETURN) #엔터

browser.find_element_by_xpath('/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-channel-renderer/div/div[2]/a/div[1]/ytd-channel-name/div/div/yt-formatted-string').click()

browser.find_element_by_xpath('//*[@class="scrollable style-scope paper-tabs"]/paper-tab[2]').click()

body = browser.find_element_by_tag_name('body')

num_of_pagedowns = 20
while num_of_pagedowns:
    body.send_keys(Keys.PAGE_DOWN)
    time.sleep(2)
    num_of_pagedowns -= 1

html0 = browser.page_source

html = BeautifulSoup(html0,'html.parser')

video_list2 = html.find_all('ytd-grid-video-renderer',{'class':'style-scope ytd-grid-renderer'})

dingo_url = []
for i in range(len(video_list2)):
    url = base_url + video_list2[i].find('a',{'id':'thumbnail'})['href']
    dingo_url.append(url)

dingo = pd.DataFrame({'name':[],
                     'thumbnail':[],
                     'view':[],
                     'previous_time':[],
                     'video_url':[],
                     'start_date':[],
                     'comment':[],
                     'likes_num':[],
                     'unlikes_num':[]})

for i in range(0,20):
    #video 제목
    name = video_list2[i].find('a',{'id':'video-title'}).text
    
    #섬네일
    thum = video_list2[i].find('a',{'id':'thumbnail'}).find('img')['src']
    
    #url 정보
    url = base_url + video_list2[i].find('a',{'id':'thumbnail'})['href']
    
    meta0 = video_list2[i].find('div',{'id':'metadata-line'})
    
    #조회수는 숫자만 추출 가능
    view = meta0.find_all('span',{'class':'style-scope ytd-grid-video-renderer'})[0].text.split()[1]
    
    previous = meta0.find_all('span',{'class':'style-scope ytd-grid-video-renderer'})[1].text
    
    start_url = dingo_url[i]
    browser.get(start_url)
    body = browser.find_element_by_tag_name('body')
    
    time.sleep(25) #로딩시간 설정
    
    num_of_pagedowns = 2
    while num_of_pagedowns:
        body.send_keys(Keys.PAGE_DOWN)
        time.sleep(3)
        num_of_pagedowns -= 1
    
    html0 = browser.page_source
    html = BeautifulSoup(html0,'html.parser')
    
    start_date = html.find('div',{'id':'date'}).text
    comment = html.find('h2',{'id':'count'}).find('yt-formatted-string').text
    
    #좋아요수
    likes_num = html.find('yt-formatted-string',{'id':'text','class':'style-scope ytd-toggle-button-renderer style-text',
                                                'aria-label': re.compile('좋아요')}).text+'개'
    
    unlikes_num = html.find('yt-formatted-string',{'id':'text','class':'style-scope ytd-toggle-button-renderer style-text',
                                                'aria-label': re.compile('싫어요')}).text+'개'
    
    insert_data = pd.DataFrame({'name':[name],
                               'thumbnail':[thum],
                               'view': [view],
                               'previous_time' : [previous],
                               'video_url':[url],
                               'start_date':[start_date],
                               'comment':[comment],
                               'likes_num':[likes_num],
                               'unlikes_num':[unlikes_num]})
    dingo = dingo.append(insert_data)

dingo = dingo.reset_index(drop=True)

dingo.drop('index',axis=1)