Data Parsing

import re
import requests

headers = {"User-Agent": "Mozilla/5.0"}
html = requests.get("https://example.com", headers=headers, timeout=10).text

# Extract all image URLs
pattern = r'<img[^>]+src="([^"]+)"'
img_urls = re.findall(pattern, html)
print(img_urls[:5])

pip install beautifulsoup4 lxml

import requests
from bs4 import BeautifulSoup

headers = {"User-Agent": "Mozilla/5.0"}
resp = requests.get("https://books.toscrape.com/", headers=headers, timeout=10)
resp.raise_for_status()

# lxml parser is recommended: fast and forgiving of malformed HTML
soup = BeautifulSoup(resp.text, "lxml")

# Access the first matching tag directly
title = soup.title              # <title>...</title>
first_h1 = soup.h1             # First h1 on the page

# find: returns the first matching tag
tag = soup.find("div", class_="product_pod")
tag = soup.find("a", href=True)

# find_all: returns a list of all matching tags
articles = soup.find_all("article", class_="product_pod")
links = soup.find_all("a", limit=10)   # At most 10 results

# select: CSS selector (recommended, very flexible)
items = soup.select("article.product_pod")           # class selector
prices = soup.select(".price_color")                 # class
header = soup.select_one("div#page_header")          # id, returns first match
nested = soup.select("div.content > ul > li > a")    # hierarchical

for article in soup.select("article.product_pod"):
    # Text extraction
    title = article.h3.a["title"]           # Attribute value
    price = article.select_one(".price_color").get_text(strip=True)
    rating = article.p["class"][1]          # List attribute, take second element

    # tag.string: direct text (only valid when there is a single child text node)
    # tag.get_text(): combined text of all descendants (strip=True removes whitespace)
    print(f"{title}: {price} (rating: {rating})")

import requests
from bs4 import BeautifulSoup
from pathlib import Path

headers = {"User-Agent": "Mozilla/5.0"}
base_url = "https://books.toscrape.com/catalogue/page-{}.html"

results = []

for page in range(1, 4):   # Scrape the first 3 pages
    url = base_url.format(page) if page > 1 else "https://books.toscrape.com/"
    resp = requests.get(url, headers=headers, timeout=10)
    resp.raise_for_status()

    soup = BeautifulSoup(resp.text, "lxml")

    for article in soup.select("article.product_pod"):
        results.append({
            "title": article.h3.a["title"],
            "price": article.select_one(".price_color").get_text(strip=True),
            "availability": article.select_one(".availability").get_text(strip=True),
        })

print(f"Total books scraped: {len(results)}")
for book in results[:5]:
    print(book)

pip install lxml

import requests
from lxml import etree

headers = {"User-Agent": "Mozilla/5.0"}
resp = requests.get("https://books.toscrape.com/", headers=headers, timeout=10)
resp.encoding = "utf-8"

# Build an etree object from the HTML string
tree = etree.HTML(resp.text)

# Global search
articles = tree.xpath('//article[@class="product_pod"]')

for article in articles:
    # Local parsing: use . to represent the current node
    title_nodes = article.xpath('.//h3/a/@title')
    price_nodes = article.xpath('.//p[@class="price_color"]/text()')

    if title_nodes and price_nodes:
        print(f"{title_nodes[0]}: {price_nodes[0].strip()}")

from lxml import etree

# Parse a local file
tree = etree.parse("local.html")   # use parse()
results = tree.xpath('//div[@class="item"]/text()')
print(results)