robots-dot-txt

A simple crawler made in JavaScript for Node.

Installation

1$ npm install robots-dot-txt

This should install the package in your project.

Usage

1const { RobotsParser } = require("../index.js");
2const robots = `
3# this is a comment
4User-agent: Googlebot
5Disallow: /nogoogle/*
6Allow: /nobing/*
7
8User-agent: Bingbot
9Disallow: /nobing/*
10    
11User-agent: *
12Disallow: /tmp/*`;
13
14robotsParser = new RobotsParser(robots);
15console.log(robotsParser.parse());
16// Output:
17// {
18//   Googlebot: { disallow: [ '/nogoogle/*' ], allow: [ '/nobing/*' ] },
19//   Bingbot: { disallow: [ '/nobing/*' ], allow: [] },
20//   '*': { disallow: [ '/tmp/*' ], allow: [] }
21// }
22
23console.log(robotsParser.canAccess("Googlebot", "/nogoogle/test")) // false
24console.log(robotsParser.canAccess("Googlebot", "/some/path")) // true
25console.log(robotsParser.canAccess("NotARealUA", "/some/path")) // true