> ## Documentation Index
> Fetch the complete documentation index at: https://docs.devin.ai/llms.txt
> Use this file to discover all available pages before exploring further.

# Web Scraping & Automation

export const PromptBlock = ({children, type, agent, intent, playbookId}) => {
  var utm = 'utm_source=docs&utm_medium=use-case-gallery&utm_campaign=prompt-block';
  var tag = 'docs-use-case-gallery';
  var agentParams = (agent ? '&agent=' + agent : '') + (intent ? '&intent=' + intent : '') + (playbookId ? '&playbookId=' + playbookId : '');
  var label = type === 'schedule' ? 'Schedule in Devin' : type === 'playbook' ? 'Create Playbook' : type === 'knowledge' ? 'Add to Knowledge' : agent === 'advanced' ? 'Try in Devin' : agent === 'dana' ? 'Try in Dana' : agent === 'ada' ? 'Try in Ask Devin' : 'Try in Devin';
  var buildUrl = function (text) {
    var encoded = encodeURIComponent(text);
    if (type === 'schedule') return 'https://app.devin.ai/settings/schedules/create?' + utm + agentParams + '&prompt=' + encoded;
    if (type === 'playbook') return 'https://app.devin.ai/settings/playbooks/create?' + utm + '&body=' + encoded;
    if (type === 'knowledge') return 'https://app.devin.ai/knowledge?' + utm + '&body=' + encoded;
    if (agent === 'ada') return 'https://app.devin.ai/search?' + utm + '&noSubmit=true&prompt=' + encoded;
    return 'https://app.devin.ai/?tags=' + tag + '&' + utm + agentParams + '&prompt=' + encoded;
  };
  const ref = React.useRef(null);
  const [href, setHref] = React.useState('#');
  React.useEffect(() => {
    if (!ref.current) return;
    var codeEl = ref.current.querySelector('pre code');
    if (codeEl) {
      var text = codeEl.textContent.trim();
      if (text) setHref(buildUrl(text));
    }
    var header = ref.current.querySelector('[data-component-part="code-block-header"]');
    if (header && !header.querySelector('.prompt-block-devin-link')) {
      var link = document.createElement('a');
      link.href = href;
      link.target = '_blank';
      link.rel = 'noopener noreferrer';
      link.className = 'prompt-block-devin-link';
      link.style.cssText = 'display:inline-flex;align-items:center;gap:6px;text-decoration:none;color:#fff;font-size:11px;font-weight:500;padding:4px 10px;border-radius:6px;white-space:nowrap;background:#317CFF;transition:background 0.2s;margin-left:8px;';
      link.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"/><polyline points="15 3 21 3 21 9"/><line x1="10" y1="14" x2="21" y2="3"/></svg> ' + label;
      link.onmouseenter = function () {
        link.style.background = '#2968D9';
      };
      link.onmouseleave = function () {
        link.style.background = '#317CFF';
      };
      header.appendChild(link);
    }
    var existingLink = ref.current.querySelector('.prompt-block-devin-link');
    if (existingLink && href !== '#') existingLink.href = href;
  });
  return <div className="prompt-block" ref={ref}>{children}</div>;
};

## Overview

Devin is your tireless web scraping assistant. It can build web scrapers or even do repetitive web research and information gathering tasks itself!

## Use Cases

1. Web scraping and data collection
2. Automated data extraction
3. Converting scraped data to structured formats
4. Handling both static and dynamic web content
5. Browser automation for repetitive tasks
6. Building automated data collection pipelines using the [API Reference](/api-reference/overview)

## Example Prompts

<AccordionGroup>
  <Accordion title="Scrape emojis" icon="face-smile">
    <PromptBlock>
      ```txt Scrape emojis theme={null}
      Use this (https://github.com/muan/unicode-emoji-json) to write a function that converts a string like https://www.gstatic.com/android/keyboard/emojikitchen/20201001/u1f600/u1f600_u2615.png to "grinning_face_warm_beverage" by extracting the 2 emojis (u1f600, u2615) and converting them to actual emojis.
      ```
    </PromptBlock>
  </Accordion>

  <Accordion title="Scraping with client-side and server-side scripts" icon="shovel-snow">
    <PromptBlock>
      ```txt Scrape website theme={null}
      ## Overview
      This playbook can be used to scrape a website and return the results to the user, along with the client-side and server-side scraping scripts used to generate the results.
      ```
    </PromptBlock>
  </Accordion>

  <Accordion title="Repetitive web tasks" icon="globe">
    <PromptBlock>
      ```txt Download logos theme={null}
      Please find and download the logos of 50 of the Fortune 500 companies.
      ```
    </PromptBlock>
  </Accordion>
</AccordionGroup>

## Example Sessions

### Emoji Data Processing

**Scrape Emojis**

Learn how to parse and convert emoji Unicode data from GitHub repositories into human-readable formats. This session demonstrates working with JSON data sources and string manipulation for emoji processing.

[https://app.devin.ai/sessions/4f8a7b129820493b9c0ca140cddede50](https://app.devin.ai/sessions/4f8a7b129820493b9c0ca140cddede50)

<iframe src="https://app.devin.ai/sessions/4f8a7b129820493b9c0ca140cddede50?embedded=true" width="100%" height="800px" />

### YouTube Content Extraction

**Scrape YouTube Playlist**

Discover how to programmatically extract video metadata from YouTube playlists. This session covers using Python to access video titles, descriptions, and other playlist information while respecting YouTube's terms of service.

[https://app.devin.ai/sessions/8c6edbbb0bce4b70acd09255e1994c0b](https://app.devin.ai/sessions/8c6edbbb0bce4b70acd09255e1994c0b)

<iframe src="https://app.devin.ai/sessions/8c6edbbb0bce4b70acd09255e1994c0b?embedded=true" width="100%" height="800px" />

### E-commerce Data Collection

**Scrape Ebay Data**

Learn techniques for gathering product information from eBay listings at scale. This session explores automated web scraping approaches for collecting prices, descriptions, and seller information while handling pagination and rate limiting.

[https://app.devin.ai/sessions/dc70fe0649cb4041852da384e65d42be](https://app.devin.ai/sessions/dc70fe0649cb4041852da384e65d42be)

<iframe src="https://app.devin.ai/sessions/dc70fe0649cb4041852da384e65d42be?embedded=true" width="100%" height="800px" />