Web Scraping Prompt for Working Python Scrapers

```python
import requests
from bs4 import BeautifulSoup
import time
import csv
import json
import logging
import re
from urllib.parse import urljoin, urlparse
from urllib.robotparser import RobotFileParser

# Configure logging to capture debug/info/error messages
logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(message)s',
    level=logging.INFO
)

class WebScraper:
    """
    General purpose web scraper with support for:
    - Following pagination links
    - Respecting robots.txt crawl delay and disallowed paths
    - Handling network errors and retries
    - Parsing HTML with BeautifulSoup
    - Saving results to CSV or JSON
    """

    def __init__(self, base_url, start_path='/', user_agent='MyScraperBot/1.0', max_retries=3, delay=None):
        """
        Initialize scraper instance.

        :param base_url: Base url of the target website, e.g. https://example.com
        :param start_path: Initial path to start scraping from, defaults to '/'
        :param user_agent: User-Agent string to use for requests
        :param max_retries: Number of times to retry requests on failure
        :param delay: Fixed delay (seconds) between requests; if None, use robots.txt crawl-delay if available
        """
        self.base_url = base_url.rstrip('/')
        self.start_url = urljoin(self.base_url, start_path.lstrip('/'))
        self.session = requests.Session()
        self.session.headers.update({'User-Agent': user_agent})
        self.max_retries = max_retries
        self.delay = delay  # Delay between requests in seconds; overrides robots.txt if set
        self.robot_parser = RobotFileParser()
        self.robot_parser.set_url(urljoin(self.base_url, '/robots.txt'))
        self.robot_parser.read()

        # Determine crawl delay from robots.txt if delay not passed explicitly
        if delay is None:
            self.delay = self.robot_parser.crawl_delay(user_agent)
            if self.delay is None:
                # Default polite delay if none specified by robots.txt
                self.delay = 1.0

        logging.info(f"Initialized scraper for {self.base_url} with a delay of {self.delay} seconds")

    def fetch(self, url):
        """
        Fetch content at url respecting retries and errors.

        :param url: Full URL to fetch
        :return: Response text if successful, None if failure
        """
        if not self.robot_parser.can_fetch(self.session.headers['User-Agent'], url):
            logging.warning(f"Fetching disallowed by robots.txt: {url}")
            return None

        for attempt in range(1, self.max_retries + 1):
            try:
                logging.debug(f"Fetching {url} (Attempt {attempt})")
                response = self.session.get(url, timeout=10)
                response.raise_for_status()
                return response.text
            except requests.RequestException as e:
                logging.warning(f"Error fetching {url}: {e} (Attempt {attempt})")
                time.sleep(2 ** attempt)  # Exponential backoff
        logging.error(f"Failed to fetch {url} after {self.max_retries} attempts")
        return None

    def parse(self, html):
        """
        Parse HTML content with BeautifulSoup.

        Override this method based on the specific data to scrape.

        :param html: Raw HTML content as string
        :return: Data extracted (list of dict) and next page url (or None)
        """
        soup = BeautifulSoup(html, 'html.parser')

        # Placeholder parsing logic:
        # Example: Scrape title and URL of items in a list, and detect "next page" link
        data = []

        # Assuming the data are in elements with class 'item', customize as needed
        items = soup.select('.item')
        for item in items:
            title_tag = item.select_one('.title')
            link_tag = item.select_one('a')
            if title_tag and link_tag and 'href' in link_tag.attrs:
                data.append({
                    'title': title_tag.get_text(strip=True),
                    'url': urljoin(self.base_url, link_tag['href'])
                })

        # Find next page link - assuming rel="next" or a link with text "Next"
        next_page_url = None
        next_link = soup.find('a', rel='next')
        if not next_link:
            # Try link text "Next"
            next_link = soup.find('a', string=re.compile(r'next', re.I))

        if next_link and 'href' in next_link.attrs:
            next_page_url = urljoin(self.base_url, next_link['href'])

        return data, next_page_url

    def scrape_all(self):
        """
        Scrape all pages starting from start_url, handling pagination.

        :return: Full list of data dictionaries scraped
        """
        current_url = self.start_url
        all_data = []

        while current_url:
            logging.info(f"Scraping page: {current_url}")
            html = self.fetch(current_url)
            if html is None:
                logging.error(f"Skipping page due to fetch failure: {current_url}")
                break

            data, next_page_url = self.parse(html)
            logging.info(f"Extracted {len(data)} items from {current_url}")
            all_data.extend(data)

            if next_page_url == current_url:
                logging.warning("Next page URL is same as current, stopping to prevent loop")
                break

            current_url = next_page_url

            if current_url:
                logging.debug(f"Sleeping for {self.delay} seconds before next request")
                time.sleep(self.delay)

        logging.info(f"Scraping complete. Total items collected: {len(all_data)}")
        return all_data

    def save_to_csv(self, data, filename):
        """
        Save list of dictionaries to CSV.

        :param data: List of dicts
        :param filename: Output CSV file path
        """
        if not data:
            logging.warning("No data to save to CSV.")
            return

        # Extract headers from keys of first dict
        headers = data[0].keys()
        try:
            with open(filename, 'w', newline='', encoding='utf-8') as f:
                writer = csv.DictWriter(f, fieldnames=headers)
                writer.writeheader()
                writer.writerows(data)
            logging.info(f"Data saved to CSV file {filename}")
        except Exception as e:
            logging.error(f"Error saving to CSV: {e}")

    def save_to_json(self, data, filename):
        """
        Save list of dictionaries to JSON file.

        :param data: List of dicts
        :param filename: Output JSON file path
        """
        try:
            with open(filename, 'w', encoding='utf-8') as f:
                json.dump(data, f, ensure_ascii=False, indent=4)
            logging.info(f"Data saved to JSON file {filename}")
        except Exception as e:
            logging.error(f"Error saving to JSON: {e}")

if __name__ == '__main__':
    """
    Example usage of WebScraper class.
    Replace `base_url