使用selenium爬取淘B商品列表信息入库MongoDb
2021/8/21 19:06:17
本文主要是介绍使用selenium爬取淘B商品列表信息入库MongoDb,对大家解决编程问题具有一定的参考价值,需要的程序猿们随着小编来一起学习吧!
selenium_taobao_com.py
#!/usr/bin/env python3 # coding=utf-8 # Version:python3.6.1 # File:selenium_taobao_com.py # Author:LGSP_Harold from urllib.parse import quote import pymongo from selenium import webdriver from selenium.common.exceptions import TimeoutException from selenium.webdriver import ActionChains from selenium.webdriver.common.by import By from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from pyquery import PyQuery as pq # firefox_options = webdriver.FirefoxOptions() # firefox_options.add_argument('--headless') # browser = webdriver.Firefox(firefox_options=firefox_options) browser = webdriver.Firefox() wait = WebDriverWait(browser, 10) client = pymongo.MongoClient('mongodb://admin:admin@localhost:27017') db = client.db_taobao_com def index_page(page): """ 抓取索引页 :param page:页码 """ print('正在爬取第', page, '页') try: if page > 1: input = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager div.form > input'))) submit = WebDriverWait(browser, 60, 3).until( EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager div.form span.btn.J_Submit'))) input.clear() input.send_keys(page) submit.click() wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR, '#mainsrp-pager li.item.active > span'), str(page))) wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.m-itemlist .items .item'))) next_page = browser.find_element_by_xpath('//li[@class="item next"]') js4 = 'arguments[0].scrollIntoView();' browser.execute_script(js4, next_page) get_products() except TimeoutException: index_page(page) def get_products(): """ 获取商品数据 """ html = browser.page_source doc = pq(html) items = doc('#mainsrp-itemlist .items .item').items() for item in items: product = { 'image': item.find('.pic .img').attr('data-src'), 'price': item.find('.price').text(), 'deal': item.find('.deal-cnt').text(), 'title': item.find('.title').text(), 'shop': item.find('.shop').text(), 'location': item.find('.location').text() } print(product) save_to_mongo(product) def save_to_mongo(result): """ 保存至MongoDB :param result:结果 """ try: if db.collection_product.insert_one(result): print('存储到MongoDB成功') except Exception as e: print('存储到MongoDB失败') print(e) def login(): url = 'https://login.taobao.com/member/login.jhtml' browser.get(url=url) # 淘宝反爬机制会检测到selenium,无法使用账号密码登录(登录失败原因,验证码验证失败) # 破解方法:使用淘宝APP扫码登录 # username = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#fm-login-id'))) # password = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#fm-login-password'))) # submit = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.fm-btn > button'))) # # username.clear() # user = input('输入会员名/邮箱/手机号:') # password.clear() # pwd = input('输入登录密码:') # # username.send_keys(user) # password.send_keys(pwd) # submit.click() try: qr_code = WebDriverWait(browser, 30, 5).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'i.iconfont.icon-qrcode'))) qr_code.click() # above = browser.find_element_by_class_name('i.iconfont.icon-qrcode') # ActionChains(browser).click(above).perform() print('请用淘宝APP扫码登录') if WebDriverWait(browser, 60, 5).until(EC.presence_of_element_located((By.CSS_SELECTOR, '.site-nav-login-info-nick'))): url_index = WebDriverWait(browser, 60, 3).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div.site-nav-menu-hd > a > span'))) url_index.click() except TimeoutException: print('登录超时') browser.quit() def main(): login() """ 遍历每一页 """ goods = input('输入您要搜索的商品:') page = int(input('输入您要爬取的总页数:')) url = 'https://s.taobao.com/search?q=' + quote(goods) browser.get(url=url) for i in range(1, page + 1): index_page(i) browser.quit() if __name__ == '__main__': main()
这篇关于使用selenium爬取淘B商品列表信息入库MongoDb的文章就介绍到这儿,希望我们推荐的文章对大家有所帮助,也希望大家多多支持为之网!
- 2024-09-26通过 gcloud CLI 认证从本地脚本写入 Google Sheets
- 2024-09-24GoLand 新建项目 Enable vendoring support automatically 的作用是什么?-icode9专业技术文章分享
- 2024-09-21MongoDB资料:新手入门与初级应用指南
- 2024-09-20MongoDB教程:初学者必备指南
- 2024-09-05MongoDB入门:快速掌握NoSQL数据库基础
- 2024-08-28go 项目中怎么打印调试-icode9专业技术文章分享
- 2024-08-21swoole未来的发展前景与golang对比哪个更好-icode9专业技术文章分享
- 2024-08-16goland 已经下了中文插件了, 怎么设置成中文-icode9专业技术文章分享
- 2024-07-26使用 SendGrid 的 Go 客户端库能同时给多个邮箱发吗-icode9专业技术文章分享
- 2024-07-26使用 SendGrid 的 Go 客户端库时怎么设置header 和 标签tag 呢-icode9专业技术文章分享