import requests
from lxml import etree

def scrape_website(url):
# 发送http请求
headers = {
'user-agent': 'mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/91.0.4472.124 safari/537.36'
}
response = requests.get(url, headers=headers)

# 解析html
html = etree.html(response.content)

# 使用xpath提取信息
title = html.xpath('//title/text()')[0]
paragraphs = html.xpath('//p/text()')
links = html.xpath('//a/@href')

return {
'title': title,
'paragraphs': paragraphs,
'links': links
}

# 使用示例
result = scrape_website('https://example.com')
print(f"title: {result['title']}")
print(f"number of paragraphs: {len(result['paragraphs'])}")
print(f"number of links: {len(result['links'])}")

# 选择所有包含"news"类的div元素
news_divs = html.xpath('//div[contains(@class, "news")]')

# 选择第一个段落
first_paragraph = html.xpath('//p[1]/text()')

# 选择所有class为"content"且id为"main"的div元素
content_divs = html.xpath('//div[@class="content" and @id="main"]')

# 选择所有具有子元素p的div元素
divs_with_p = html.xpath('//div[child::p]')

# 选择所有具有属性class的元素的父元素
parents = html.xpath('//*[@class]/parent::*')

import requests
from lxml import etree
import csv

def scrape_news_website(url):
headers = {
'user-agent': 'mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/91.0.4472.124 safari/537.36'
}
response = requests.get(url, headers=headers)
html = etree.html(response.content)

# 提取新闻文章
articles = []
for article in html.xpath('//div[@class="article"]'):
title = article.xpath('.//h2/text()')[0]
summary = article.xpath('.//p[@class="summary"]/text()')[0]
author = article.xpath('.//span[@class="author"]/text()')[0]
date = article.xpath('.//span[@class="date"]/text()')[0]
link = article.xpath('.//a[@class="read-more"]/@href')[0]

articles.append({
'title': title,
'summary': summary,
'author': author,
'date': date,
'link': link
})

return articles

def save_to_csv(articles, filename):
with open(filename, 'w', newline='', encoding='utf-8') as csvfile:
fieldnames = ['title', 'summary', 'author', 'date', 'link']
writer = csv.dictwriter(csvfile, fieldnames=fieldnames)

writer.writeheader()
for article in articles:
writer.writerow(article)

# 主函数
def main():
url = 'https://example-news-site.com'
articles = scrape_news_website(url)
save_to_csv(articles, 'news_articles.csv')
print(f"scraped {len(articles)} articles and saved to news_articles.csv")

if __name__ == '__main__':
main()

from selenium import webdriver
from selenium.webdriver.chrome.options import options
from lxml import etree
import time

def scrape_dynamic_website(url):
chrome_options = options()
chrome_options.add_argument("--headless") # 无头模式

driver = webdriver.chrome(options=chrome_options)
driver.get(url)

# 等待页面加载（你可能需要调整等待时间）
time.sleep(5)

# 获取渲染后的html
html_content = driver.page_source
driver.quit()

# 解析html
html = etree.html(html_content)

# 使用xpath提取信息
# ...（根据具体网站结构编写xpath）

return extracted_data

# 使用示例
data = scrape_dynamic_website('https://example-dynamic-site.com')