proxy

Suggest edits
  • __all__ = ['ProxyWebScraperDriver'] module-attribute

Bases: BaseWebScraperDriver

Source Code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py
@define
class ProxyWebScraperDriver(BaseWebScraperDriver):
    proxies: dict = field(kw_only=True, metadata={"serializable": False})
    params: dict = field(default=Factory(dict), kw_only=True, metadata={"serializable": True})

    def fetch_url(self, url: str) -> str:
        response = requests.get(url, proxies=self.proxies, **self.params)

        return response.text

    def extract_page(self, page: str) -> TextArtifact:
        return TextArtifact(page)

  • params = field(default=Factory(dict), kw_only=True, metadata={'serializable': True}) class-attribute instance-attribute

  • proxies = field(kw_only=True, metadata={'serializable': False}) class-attribute instance-attribute

extract_page(page)

Source Code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py
def extract_page(self, page: str) -> TextArtifact:
    return TextArtifact(page)

fetch_url(url)

Source Code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py
def fetch_url(self, url: str) -> str:
    response = requests.get(url, proxies=self.proxies, **self.params)

    return response.text
← Prev

proxy_web_scraper_driver

↑ Up

web_scraper

Next →

trafilatura_web_scraper_driver

Could this page be better? Report a problem or suggest an addition!