Complete Guide to Scraping Airbnb Data Using Python

import requests
from bs4 import BeautifulSoup
import time
import random

# Configure headers to mimic a browser
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    'Accept-Language': 'en-US,en;q=0.9',
    'Referer': 'https://www.google.com/'
}

def scrape_airbnb_search(location, max_pages=3):
    base_url = f"https://www.airbnb.com/s/{location}/homes"
    all_listings = []
    
    for page in range(1, max_pages + 1):
        params = {
            'items_offset': (page - 1) * 20,
            'section_offset': 3
        }
        
        try:
            # Add random delay to avoid detection
            time.sleep(random.uniform(1, 3))
            
            response = requests.get(base_url, headers=HEADERS, params=params)
            response.raise_for_status()
            
            soup = BeautifulSoup(response.text, 'html.parser')
            listings = soup.find_all('div', {'itemprop': 'itemListElement'})
            
            for listing in listings:
                listing_data = {
                    'title': get_text_or_none(listing, 'div.t1jojoys'),
                    'price': get_text_or_none(listing, 'span._tyxjp1'),
                    'rating': get_text_or_none(listing, 'span.r1dxllyb'),
                    'reviews': get_text_or_none(listing, 'span._a7a5sx'),
                    'link': "https://www.airbnb.com" + listing.find('a')['href'] if listing.find('a') else None
                }
                all_listings.append(listing_data)
                
        except Exception as e:
            print(f"Error scraping page {page}: {str(e)}")
            continue
            
    return all_listings

def get_text_or_none(element, selector):
    found = element.select_one(selector)
    return found.text.strip() if found else None

# Example usage
if __name__ == "__main__":
    listings = scrape_airbnb_search("New-York--NY--USA", max_pages=2)
    print(f"Scraped {len(listings)} listings")

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
import time
import random

def setup_driver():
    options = Options()
    options.add_argument("--headless")
    options.add_argument("--disable-blink-features=AutomationControlled")
    options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
    driver = webdriver.Chrome(options=options)
    return driver

def scrape_airbnb_selenium(location, max_scrolls=3):
    driver = setup_driver()
    url = f"https://www.airbnb.com/s/{location}/homes"
    driver.get(url)
    
    try:
        # Accept cookies if popup appears
        WebDriverWait(driver, 10).until(
            EC.element_to_be_clickable((By.XPATH, '//button[contains(text(), "Accept")]'))
        ).click()
    except:
        pass
    
    # Scroll to load more listings
    for _ in range(max_scrolls):
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(random.uniform(2, 4))
    
    # Wait for listings to load
    WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, 'div.c4mnd7m'))
    )
    
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    driver.quit()
    
    # Process listings as before
    listings = []
    for listing in soup.select('div[itemprop="itemListElement"]'):
        listings.append({
            'title': get_text_or_none(listing, 'div.t1jojoys'),
            'price': get_text_or_none(listing, 'span._tyxjp1'),
            # Add more fields as needed
        })
    
    return listings

def scrape_listing_details(listing_url):
    driver = setup_driver()
    driver.get(listing_url)
    
    try:
        # Wait for main content to load
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, 'div._1m8bb6v'))
        )
        
        # Extract detailed information
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        
        details = {
            'description': get_text_or_none(soup, 'div._1n81at5'),
            'bedrooms': get_text_or_none(soup, 'span._faldii7'),
            'bathrooms': get_text_or_none(soup, 'span._1n81at5'),
            'amenities': [a.text for a in soup.select('div._1gw6tte')],
            'house_rules': get_text_or_none(soup, 'div._1y6fhhr'),
            'cancellation_policy': get_text_or_none(soup, 'div._1d8yint'),
            'host_name': get_text_or_none(soup, 'div._1m8bb6v'),
            'host_response_rate': get_text_or_none(soup, 'span._1d8yint'),
            'coordinates': extract_coordinates(soup)
        }
        
        return details
        
    finally:
        driver.quit()

import csv
from datetime import datetime

def save_to_csv(data, filename_prefix="airbnb"):
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    filename = f"{filename_prefix}_{timestamp}.csv"
    
    fieldnames = [
        'title', 'price', 'rating', 'reviews', 'location',
        'room_type', 'link', 'scraped_at'
    ]
    
    with open(filename, 'w', newline='', encoding='utf-8') as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        
        for listing in data:
            listing['scraped_at'] = timestamp
            writer.writerow(listing)
    
    print(f"Saved {len(data)} listings to {filename}")

import sqlite3
from contextlib import contextmanager

@contextmanager
def get_db_connection():
    conn = sqlite3.connect('airbnb_data.db')
    conn.row_factory = sqlite3.Row
    try:
        yield conn
    finally:
        conn.close()

def init_database():
    with get_db_connection() as conn:
        conn.execute('''
            CREATE TABLE IF NOT EXISTS listings (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                title TEXT,
                price TEXT,
                rating REAL,
                reviews INTEGER,
                location TEXT,
                room_type TEXT,
                link TEXT UNIQUE,
                scraped_at TIMESTAMP,
                raw_data TEXT
            )
        ''')
        conn.commit()

def save_to_database(listings):
    init_database()
    
    with get_db_connection() as conn:
        for listing in listings:
            conn.execute('''
                INSERT OR IGNORE INTO listings 
                (title, price, rating, reviews, location, room_type, link, scraped_at, raw_data)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
            ''', (
                listing.get('title'),
                listing.get('price'),
                listing.get('rating'),
                listing.get('reviews'),
                listing.get('location'),
                listing.get('room_type'),
                listing.get('link'),
                datetime.now().isoformat(),
                str(listing)  # Store raw data as JSON string
            ))
        conn.commit()

from selenium.webdriver.common.action_chains import ActionChains
import random

def human_like_interaction(driver):
    """Simulate human-like mouse movements"""
    actions = ActionChains(driver)
    
    # Random mouse movements
    for _ in range(random.randint(2, 5)):
        x_offset = random.randint(-50, 50)
        y_offset = random.randint(-50, 50)
        actions.move_by_offset(x_offset, y_offset)
        actions.pause(random.uniform(0.1, 0.5))
    
    actions.perform()

def avoid_detection(driver):
    # Remove webdriver properties
    driver.execute_script(
        "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
    )
    
    # Change viewport size
    driver.set_window_size(
        random.randint(1200, 1400),
        random.randint(800, 1000)
    )
    
    # Human-like interaction
    human_like_interaction(driver)

def use_proxy(driver, proxy):
    options = webdriver.ChromeOptions()
    options.add_argument(f'--proxy-server={proxy}')
    return webdriver.Chrome(options=options)

import itertools
import random

class ProxyRotator:
    def __init__(self, proxy_list):
        self.proxies = proxy_list
        self.proxy_cycle = itertools.cycle(self.proxies)
        
    def get_proxy(self):
        return next(self.proxy_cycle)
        
    def get_random_proxy(self):
        return random.choice(self.proxies)

# Example usage
proxies = [
    'http://user:pass@proxy1:port',
    'http://user:pass@proxy2:port',
    # Add more proxies
]

rotator = ProxyRotator(proxies)

def make_request_with_proxy(url):
    proxy = rotator.get_proxy()
    try:
        response = requests.get(url, proxies={'http': proxy, 'https': proxy})
        return response
    except:
        # Fallback to another proxy
        proxy = rotator.get_random_proxy()
        response = requests.get(url, proxies={'http': proxy, 'https': proxy})
        return response