url = "https://www.example.com"
response = request.urlopen(url)
html = response.read().decode('utf-8')
print(html)

data = parse.urlencode({'key1': 'value1', 'key2': 'value2'}).encode('utf-8')
req = request.request(url, data=data, method='post')
response = request.urlopen(req)
html = response.read().decode('utf-8')
print(html)

headers = {
'user-agent': 'mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/91.0.4472.124 safari/537.36'
}
req = request.request(url, headers=headers)
response = request.urlopen(req)

print(response.getheaders())
print(response.getheader('content-type'))

encoded_url = parse.quote('https://example.com/path with spaces')
print(encoded_url)

decoded_url = parse.unquote('https://example.com/path%20with%20spaces')
print(decoded_url)

params = {'key1': 'value1', 'key2': 'value2'}
query_string = parse.urlencode(params)
url = f"https://example.com/search?{query_string}"
print(url)

try:
response = request.urlopen("https://www.example.com/nonexistent")
except error.httperror as e:
print(f"http error: {e.code}")
except error.urlerror as e:
print(f"url error: {e.reason}")

proxy_handler = request.proxyhandler({'http': 'http://127.0.0.1:8080'})
opener = request.build_opener(proxy_handler)
request.install_opener(opener)
response = request.urlopen(url)

import http.cookiejar

cookie_jar = http.cookiejar.cookiejar()
opener = request.build_opener(request.httpcookieprocessor(cookie_jar))
request.install_opener(opener)
response = request.urlopen(url)

for cookie in cookie_jar:
print(f"{cookie.name}: {cookie.value}")

class customhttphandler(request.httphandler):
def http_request(self, req):
req.add_header('custom-header', 'customvalue')
return super().http_request(req)

opener = request.build_opener(customhttphandler)
request.install_opener(opener)
response = request.urlopen(url)

class noredirecthandler(request.httpredirecthandler):
def http_error_302(self, req, fp, code, msg, headers):
return fp

opener = request.build_opener(noredirecthandler)
request.install_opener(opener)
try:
response = request.urlopen(url)
except error.httperror as e:
if e.code == 302:
print(f"redirect to: {e.headers['location']}")

import asyncio
import aiohttp

async def fetch(url):
async with aiohttp.clientsession() as session:
async with session.get(url) as response:
return await response.text()

async def main():
urls = ['https://example.com', 'https://example.org', 'https://example.net']
tasks = [asyncio.create_task(fetch(url)) for url in urls]
results = await asyncio.gather(*tasks)
for url, html in zip(urls, results):
print(f"content length from {url}: {len(html)}")

asyncio.run(main())

from urllib import request
from bs4 import beautifulsoup

def scrape_website(url):
headers = {
'user-agent': 'mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/91.0.4472.124 safari/537.36'
}
req = request.request(url, headers=headers)

try:
response = request.urlopen(req)
html = response.read().decode('utf-8')

soup = beautifulsoup(html, 'html.parser')

# 提取标题
title = soup.title.string if soup.title else 'no title found'

# 提取所有段落文本
paragraphs = [p.text for p in soup.find_all('p')]

# 提取所有链接
links = [a['href'] for a in soup.find_all('a', href=true)]

return {
'title': title,
'paragraphs': paragraphs,
'links': links
}
except error.httperror as e:
print(f"http error: {e.code}")
except error.urlerror as e:
print(f"url error: {e.reason}")

return none

# 使用示例
result = scrape_website('https://example.com')
if result:
print(f"title: {result['title']}")
print(f"number of paragraphs: {len(result['paragraphs'])}")
print(f"number of links: {len(result['links'])}")