Cyborg.txt: robots.txt stuff for Node

A collection of utilities for all your robots.txt needs, like building a web crawler or your own robots-serving website.

Installation

npm install cyborg.txt

For webcrawlers

1var robots = require("cyborg.txt");
2
3// Given a URL, where's the corresponding robots.txt file?
4robots.url("http://example.com"); // => http://example.com/robots.txt
5robots.url("http://example.com/some/path/to/file.jpg"); // => http://example.com/robots.txt

For webmasters

1var robots = require('cyborg.txt');
2
3// Build the body of a robots.txt
4
5robots.build({
6  '*': ['/secrets'],
7  'another-web-crawler': ['cool_folder/hiddenA'], // you can ignore the slash at the start
8  'let-everything-through-crawler': [] // allow anything
9  'CoolSearchEngine': ['/hidden_picture.jpg', '/something_else.txt'],
10});
11
12  /*
13  User-agent: *
14  Disallow: /secrets
15
16  User-agent: CoolSearchEngine
17  Disallow: /hidden_picture.jpg
18
19  User-agent: let-everything-through-crawler
20  Disallow:
21
22  User-agent: another-web-crawler
23  Disallow: /cool_folder/hiddenA
24  Disallow: /cool_folder/hidden
25
26  */
27
28robots.build(); // allow anything through
29
30  /*
31  User-agent: *
32  Disallow:
33  */
34
35// Use as Connect middleware (works with Express)
36
37var connect = require('connect');
38var app = connect();
39app.use(robots.middleware({  // takes same options as robots.build
40  '*': ['/secret_lair']
41});
42app.listen(1337);