How to Scrape Business Listings from Google Using Python

import requests
from bs4 import BeautifulSoup
import urllib.parse
import time

def search_google_business(query, location=None, num_results=10):
    base_url = "https://www.google.com/search"
    
    params = {
        "q": query,
        "num": num_results,
        "tbm": "lcl"  # Local business results
    }
    
    if location:
        params["near"] = location
    
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
        "Accept-Language": "en-US,en;q=0.9"
    }
    
    try:
        response = requests.get(base_url, params=params, headers=headers)
        response.raise_for_status()
        
        soup = BeautifulSoup(response.text, 'html.parser')
        business_results = []
        
        # Extract local business listings
        listings = soup.find_all('div', class_='VkpGBb')
        
        for listing in listings:
            business_data = {
                'name': get_text_or_none(listing, 'div.dbg0pd'),
                'rating': get_text_or_none(listing, 'span.btQScd'),
                'reviews': get_text_or_none(listing, 'span.rllt__details span:nth-of-type(2)'),
                'address': get_text_or_none(listing, 'span.rllt__details span:nth-of-type(1)'),
                'phone': get_text_or_none(listing, 'span.rllt__details span:nth-of-type(3)'),
                'link': "https://www.google.com" + listing.find('a')['href'] if listing.find('a') else None
            }
            business_results.append(business_data)
            
        return business_results
        
    except Exception as e:
        print(f"Error scraping Google: {str(e)}")
        return []

def get_text_or_none(element, selector):
    found = element.select_one(selector)
    return found.text.strip() if found else None

# Example usage
if __name__ == "__main__":
    businesses = search_google_business("restaurants", "New York", 5)
    for biz in businesses:
        print(f"{biz['name']} - {biz['rating']} ({biz['reviews']})")

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
import time

def setup_driver():
    options = Options()
    options.add_argument("--headless")
    options.add_argument("--disable-blink-features=AutomationControlled")
    options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
    driver = webdriver.Chrome(options=options)
    return driver

def scrape_google_business_selenium(query, location=None):
    driver = setup_driver()
    url = f"https://www.google.com/search?q={urllib.parse.quote(query)}&tbm=lcl"
    
    if location:
        url += f"&near={urllib.parse.quote(location)}"
    
    driver.get(url)
    
    try:
        # Wait for business listings to load
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, 'div.VkpGBb'))
        )
        
        # Scroll to load more results
        for _ in range(3):
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(2)
        
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        return parse_business_listings(soup)
        
    finally:
        driver.quit()

def parse_business_listings(soup):
    listings = soup.find_all('div', class_='VkpGBb')
    business_data = []
    
    for listing in listings:
        try:
            biz = {
                'name': listing.find('div', class_='dbg0pd').text,
                'rating': listing.find('span', class_='btQScd').text,
                'reviews': listing.select_one('span.rllt__details span:nth-of-type(2)').text,
                'address': listing.select_one('span.rllt__details span:nth-of-type(1)').text,
                'link': "https://www.google.com" + listing.find('a')['href']
            }
            business_data.append(biz)
        except Exception as e:
            continue
            
    return business_data

def scrape_google_maps(query, location=None, max_results=20):
    driver = setup_driver()
    url = f"https://www.google.com/maps/search/{urllib.parse.quote(query)}/"
    
    if location:
        url += f"@{location}/"
    
    driver.get(url)
    
    try:
        # Wait for results to load
        WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, 'div.section-result'))
        )
        
        # Scroll to load more results
        scrollable_div = driver.find_element(By.CSS_SELECTOR, 'div.section-layout.section-scrollbox')
        
        for _ in range(5):
            driver.execute_script("arguments[0].scrollTop = arguments[0].scrollHeight", scrollable_div)
            time.sleep(2)
        
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        return parse_maps_listings(soup, max_results)
        
    finally:
        driver.quit()

def parse_maps_listings(soup, max_results):
    listings = soup.find_all('div', class_='section-result')[:max_results]
    business_data = []
    
    for listing in listings:
        try:
            biz = {
                'name': listing.find('h3', class_='section-result-title').text.strip(),
                'rating': listing.find('span', class_='cards-rating-score').text.strip(),
                'reviews': listing.find('span', class_='section-result-num-ratings').text.strip('()'),
                'address': listing.find('span', class_='section-result-location').text.strip(),
                'category': listing.find('span', class_='section-result-details').text.strip(),
                'link': "https://www.google.com" + listing.find('a')['href']
            }
            business_data.append(biz)
        except Exception as e:
            continue
            
    return business_data

import random
from selenium.webdriver.common.action_chains import ActionChains

def human_like_interaction(driver):
    """Simulate human-like behavior"""
    actions = ActionChains(driver)
    
    # Random mouse movements
    for _ in range(random.randint(2, 5)):
        x_offset = random.randint(-50, 50)
        y_offset = random.randint(-50, 50)
        actions.move_by_offset(x_offset, y_offset)
        actions.pause(random.uniform(0.1, 0.5))
    
    actions.perform()

def avoid_google_detection(driver):
    # Remove webdriver properties
    driver.execute_script(
        "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
    )
    
    # Randomize viewport size
    driver.set_window_size(
        random.randint(1200, 1400),
        random.randint(800, 1000)
    )
    
    # Add random delays between actions
    time.sleep(random.uniform(1, 3))
    
    # Human-like interaction
    human_like_interaction(driver)

def use_proxy_rotation(driver):
    proxy_list = [
        'http://user:pass@proxy1:port',
        'http://user:pass@proxy2:port',
        # Add more proxies
    ]
    
    proxy = random.choice(proxy_list)
    options = webdriver.ChromeOptions()
    options.add_argument(f'--proxy-server={proxy}')
    return webdriver.Chrome(options=options)