puppeteer-extra

A light-weight wrapper around puppeteer and friends to enable cool plugins through a clean interface.

Installation

1yarn add puppeteer puppeteer-extra
2# - or -
3npm install puppeteer puppeteer-extra
4
5# puppeteer-extra works with any puppeteer version:
6yarn add puppeteer@2.0.0 puppeteer-extra

Quickstart

1// puppeteer-extra is a drop-in replacement for puppeteer,
2// it augments the installed puppeteer with plugin functionality.
3// Any number of plugins can be added through `puppeteer.use()`
4const puppeteer = require('puppeteer-extra')
5
6// Add stealth plugin and use defaults (all tricks to hide puppeteer usage)
7const StealthPlugin = require('puppeteer-extra-plugin-stealth')
8puppeteer.use(StealthPlugin())
9
10// Add adblocker plugin to block all ads and trackers (saves bandwidth)
11const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker')
12puppeteer.use(AdblockerPlugin({ blockTrackers: true }))
13
14// That's it, the rest is puppeteer usage as normal 😊
15puppeteer.launch({ headless: true }).then(async browser => {
16  const page = await browser.newPage()
17  await page.setViewport({ width: 800, height: 600 })
18
19  console.log(`Testing adblocker plugin..`)
20  await page.goto('https://www.vanityfair.com')
21  await page.waitForTimeout(1000)
22  await page.screenshot({ path: 'adblocker.png', fullPage: true })
23
24  console.log(`Testing the stealth plugin..`)
25  await page.goto('https://bot.sannysoft.com')
26  await page.waitForTimeout(5000)
27  await page.screenshot({ path: 'stealth.png', fullPage: true })
28
29  console.log(`All done, check the screenshots. ✨`)
30  await browser.close()
31})

The above example uses the stealth and adblocker plugin, which need to be installed as well:

1yarn add puppeteer-extra-plugin-stealth puppeteer-extra-plugin-adblocker
2# - or -
3npm install puppeteer-extra-plugin-stealth puppeteer-extra-plugin-adblocker

If you'd like to see debug output just run your script like so:

1DEBUG=puppeteer-extra,puppeteer-extra-plugin:* node myscript.js

More examples

1import puppeteer from 'puppeteer-extra'
2
3import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker'
4import StealthPlugin from 'puppeteer-extra-plugin-stealth'
5
6puppeteer.use(AdblockerPlugin()).use(StealthPlugin())
7
8puppeteer
9  .launch({ headless: false, defaultViewport: null })
10  .then(async browser => {
11    const page = await browser.newPage()
12    await page.goto('https://bot.sannysoft.com')
13    await page.waitForTimeout(5000)
14    await page.screenshot({ path: 'stealth.png', fullPage: true })
15    await browser.close()
16  })

1const vanillaPuppeteer = require('puppeteer')
2
3const { addExtra } = require('puppeteer-extra')
4const AnonymizeUA = require('puppeteer-extra-plugin-anonymize-ua')
5
6async function main() {
7  const pptr1 = addExtra(vanillaPuppeteer)
8  pptr1.use(
9    AnonymizeUA({
10      customFn: ua => 'Hello1/' + ua.replace('Chrome', 'Beer')
11    })
12  )
13
14  const pptr2 = addExtra(vanillaPuppeteer)
15  pptr2.use(
16    AnonymizeUA({
17      customFn: ua => 'Hello2/' + ua.replace('Chrome', 'Beer')
18    })
19  )
20
21  await checkUserAgent(pptr1)
22  await checkUserAgent(pptr2)
23}
24
25main()
26
27async function checkUserAgent(pptr) {
28  const browser = await pptr.launch({ headless: true })
29  const page = await browser.newPage()
30  await page.goto('https://httpbin.org/headers', {
31    waitUntil: 'domcontentloaded'
32  })
33  const content = await page.content()
34  console.log(content)
35  await browser.close()
36}

1const { Cluster } = require('puppeteer-cluster')
2const vanillaPuppeteer = require('puppeteer')
3
4const { addExtra } = require('puppeteer-extra')
5const Stealth = require('puppeteer-extra-plugin-stealth')
6const Recaptcha = require('puppeteer-extra-plugin-recaptcha')
7
8async function main() {
9  // Create a custom puppeteer-extra instance using `addExtra`,
10  // so we could create additional ones with different plugin config.
11  const puppeteer = addExtra(vanillaPuppeteer)
12  puppeteer.use(Stealth())
13  puppeteer.use(Recaptcha())
14
15  // Launch cluster with puppeteer-extra
16  const cluster = await Cluster.launch({
17    puppeteer,
18    maxConcurrency: 2,
19    concurrency: Cluster.CONCURRENCY_CONTEXT
20  })
21
22  // Define task handler
23  await cluster.task(async ({ page, data: url }) => {
24    await page.goto(url)
25
26    const { hostname } = new URL(url)
27    const { captchas } = await page.findRecaptchas()
28    console.log(`Found ${captchas.length} captcha on ${hostname}`)
29
30    await page.screenshot({ path: `${hostname}.png`, fullPage: true })
31  })
32
33  // Queue any number of tasks
34  cluster.queue('https://bot.sannysoft.com')
35  cluster.queue('https://www.google.com/recaptcha/api2/demo')
36  cluster.queue('http://www.wikipedia.org/')
37
38  await cluster.idle()
39  await cluster.close()
40  console.log(`All done, check the screenshots. ✨`)
41}
42
43// Let's go
44main().catch(console.warn)

1import { Cluster } from 'puppeteer-cluster'
2import vanillaPuppeteer from 'puppeteer'
3
4import { addExtra } from 'puppeteer-extra'
5import Stealth from 'puppeteer-extra-plugin-stealth'
6import Recaptcha from 'puppeteer-extra-plugin-recaptcha'

1const chromium = require('chrome-aws-lambda')
2const { addExtra } = require('puppeteer-extra')
3const puppeteerExtra = addExtra(chromium.puppeteer)
4
5const launch = async () => {
6  puppeteerExtra
7    .launch({
8      args: chromium.args,
9      defaultViewport: chromium.defaultViewport,
10      executablePath: await chromium.executablePath,
11      headless: chromium.headless
12    })
13    .then(async browser => {
14      const page = await browser.newPage()
15      await page.goto('https://www.spacejam.com/archive/spacejam/movie/jam.htm')
16      await page.waitForTimeout(10 * 1000)
17      await browser.close()
18    })
19}
20
21launch() // Launch Browser

1const puppeteer = require('puppeteer-extra')
2const StealthPlugin = require('puppeteer-extra-plugin-stealth')
3puppeteer.use(StealthPlugin())
4
5const browserless = require('browserless')({ puppeteer })
6
7const saveBufferToFile = (buffer, fileName) => {
8  const wstream = require('fs').createWriteStream(fileName)
9  wstream.write(buffer)
10  wstream.end()
11}
12
13browserless
14  .screenshot('https://bot.sannysoft.com', { device: 'iPhone 6' })
15  .then(buffer => {
16    const fileName = 'screenshot.png'
17    saveBufferToFile(buffer, fileName)
18    console.log(`your screenshot is here: `, fileName)
19  })

Plugins

🔥 puppeteer-extra-plugin-stealth

• Applies various evasion techniques to make detection of puppeteer harder.

🏴 puppeteer-extra-plugin-recaptcha

• Solves reCAPTCHAs and hCaptchas automatically, using a single line of code: page.solveRecaptchas().

puppeteer-extra-plugin-adblocker

• Very fast & efficient blocker for ads and trackers. Reduces bandwidth & load times.

puppeteer-extra-plugin-devtools

• Makes puppeteer browser debugging possible from anywhere.
• Creates a secure tunnel to make the devtools frontend (incl. screencasting) accessible from the public internet

puppeteer-extra-plugin-repl

• Makes quick puppeteer debugging and exploration fun with an interactive REPL.

puppeteer-extra-plugin-block-resources

• Blocks resources (images, media, css, etc.) in puppeteer.
• Supports all resource types, blocking can be toggled dynamically.

puppeteer-extra-plugin-flash

• Allows flash content to run on all sites without user interaction.

puppeteer-extra-plugin-anonymize-ua

• Anonymizes the user-agent on all pages.
• Supports dynamic replacing, so the browser version stays intact and recent.

puppeteer-extra-plugin-user-preferences

• Allows setting custom Chrome/Chromium user preferences.
• Has itself a plugin interface which is used by e.g. puppeteer-extra-plugin-font-size.

Check out the packages folder for more plugins.

Community Plugins

These plugins have been generously contributed by members of the community. Please note that they're hosted outside the main project and not under our control or supervision.

puppeteer-extra-plugin-minmax

• Minimize and maximize puppeteer in real time.
• Great for manually solving captchas.

puppeteer-extra-plugin-portal

• Use the Chromium screencast API to remotely view and interact with puppeteer sessions.
• Great for remotely intervening when an automated task gets stuck, like captchas.

Please check the Contributing section below if you're interested in creating a plugin as well.

Contributors

Further info

Changelog

API

Table of Contents

• class: PuppeteerExtra
  • .use(plugin)
  • .launch(options?)
  • .connect(options?)
  • .defaultArgs(options?)
  • .executablePath()
  • .createBrowserFetcher(options?)
  • .plugins
  • .getPluginData(name?)
• defaultExport()
• addExtra(puppeteer)

class: PuppeteerExtra

Modular plugin framework to teach puppeteer new tricks.

This module acts as a drop-in replacement for puppeteer.

Allows PuppeteerExtraPlugin's to register themselves and to extend puppeteer with additional functionality.

Example:

1const puppeteer = require('puppeteer-extra')
2puppeteer.use(require('puppeteer-extra-plugin-anonymize-ua')())
3puppeteer.use(
4  require('puppeteer-extra-plugin-font-size')({ defaultFontSize: 18 })
5)
6;(async () => {
7  const browser = await puppeteer.launch({ headless: false })
8  const page = await browser.newPage()
9  await page.goto('http://example.com', { waitUntil: 'domcontentloaded' })
10  await browser.close()
11})()

.use(plugin)

• plugin PuppeteerExtraPlugin

Returns: this The same PuppeteerExtra instance (for optional chaining)

The main interface to register puppeteer-extra plugins.

Example:

1puppeteer.use(plugin1).use(plugin2)

• See: PuppeteerExtraPlugin

.launch(options?)

• options Puppeteer.LaunchOptions? See puppeteer docs.

Returns: Promise<Puppeteer.Browser>

The method launches a browser instance with given arguments. The browser will be closed when the parent node.js process is closed.

Augments the original puppeteer.launch method with plugin lifecycle methods.

All registered plugins that have a beforeLaunch method will be called in sequence to potentially update the options Object before launching the browser.

Example:

1const browser = await puppeteer.launch({
2  headless: false,
3  defaultViewport: null
4})

.connect(options?)

• options Puppeteer.ConnectOptions? See puppeteer docs.

Returns: Promise<Puppeteer.Browser>

Attach Puppeteer to an existing Chromium instance.

Augments the original puppeteer.connect method with plugin lifecycle methods.

All registered plugins that have a beforeConnect method will be called in sequence to potentially update the options Object before launching the browser.

.defaultArgs(options?)

• options Puppeteer.ChromeArgOptions? See puppeteer docs.

Returns: Array<string>

The default flags that Chromium will be launched with.

.executablePath()

Returns: string

Path where Puppeteer expects to find bundled Chromium.

.createBrowserFetcher(options?)

• options Puppeteer.FetcherOptions? See puppeteer docs.

Returns: Puppeteer.BrowserFetcher

This methods attaches Puppeteer to an existing Chromium instance.

.plugins

Type: Array<PuppeteerExtraPlugin>

Get a list of all registered plugins.

.getPluginData(name?)

• name string? Filter data by optional plugin name

Collects the exposed data property of all registered plugins. Will be reduced/flattened to a single array.

Can be accessed by plugins that listed the dataFromPlugins requirement.

Implemented mainly for plugins that need data from other plugins (e.g. user-preferences).

• See: PuppeteerExtraPlugin/data

defaultExport()

Type: PuppeteerExtra

The default export will behave exactly the same as the regular puppeteer (just with extra plugin functionality) and can be used as a drop-in replacement.

Behind the scenes it will try to require either puppeteer or puppeteer-core from the installed dependencies.

Example:

1// javascript import
2const puppeteer = require('puppeteer-extra')
3
4// typescript/es6 module import
5import puppeteer from 'puppeteer-extra'
6
7// Add plugins
8puppeteer.use(...)

addExtra(puppeteer)

• puppeteer VanillaPuppeteer Any puppeteer API-compatible puppeteer implementation or version.

Returns: PuppeteerExtra A fresh PuppeteerExtra instance using the provided puppeteer

An alternative way to use puppeteer-extra: Augments the provided puppeteer with extra plugin functionality.

This is useful in case you need multiple puppeteer instances with different plugins or to add plugins to a non-standard puppeteer package.

Example:

1// js import
2const puppeteerVanilla = require('puppeteer')
3const { addExtra } = require('puppeteer-extra')
4
5// ts/es6 import
6import puppeteerVanilla from 'puppeteer'
7import { addExtra } from 'puppeteer-extra'
8
9// Patch provided puppeteer and add plugins
10const puppeteer = addExtra(puppeteerVanilla)
11puppeteer.use(...)

License

Copyright © 2018 - 2023, berstend̡̲̫̹̠̖͚͓̔̄̓̐̄͛̀͘. Released under the MIT License.