Added no/robots.txt

lib/yeoman/index.js

@@ -409,6 +409,18 @@ var WordpressGenerator = yeoman.generators.Base.extend({
     }
   },
 
+  prepareHtaccess: function() {
+    try {
+      this.htaccessFile = this.readFileAsString(path.join(this.env.cwd, 'web', '.htaccess'));
+      this.htaccessFile = this.htaccessFile.replace(/# BEGIN Evolution WordPress(?:.|[\r\n]+)+?# END Evolution WordPress[\r\n]*/i, '');
+
+      this.htaccessWpBlock = !!this.htaccessFile.match(/# BEGIN WordPress(?:.|[\r\n]+)+?# END WordPress[\r\n]*/i);
+    } catch(e) {
+      this.htaccessFile = '';
+      this.htaccessWpBlock = false;
+    }
+  },
+
   prepareRoles: function() {
     this.props.roles = WordpressGenerator.roles.map(function(role) {
       role.checked = !!~this.props.roles.indexOf(role.value);

lib/yeoman/templates/web/_htaccess

@@ -1,4 +1,14 @@
-# BEGIN WordPress
+# BEGIN Evolution WordPress
+<IfModule mod_rewrite.c>
+RewriteEngine On
+RewriteBase /
+# Prevent spidering of non-canonical URLs, such as `[*.]staging.*` and `production.*`.
+RewriteCond %{HTTP_HOST} (^local|staging|^production)\.<%= props.domain.replace('.', '\\.') %>$ [NC]
+RewriteRule ^robots.txt$ no_robots.txt [L]
+</IfModule>
+# END Evolution WordPress
+<%= htaccessFile %>
+<% if (!htaccessWpBlock) { %># BEGIN WordPress
 <IfModule mod_rewrite.c>
 RewriteEngine On
 RewriteBase /
@@ -7,4 +17,4 @@ RewriteCond %{REQUEST_FILENAME} !-f
 RewriteCond %{REQUEST_FILENAME} !-d
 RewriteRule . /index.php [L]
 </IfModule>
-# END WordPress
+# END WordPress<% } %>

lib/yeoman/templates/web/no_robots.txt

@@ -0,0 +1,5 @@
+User-agent: *
+
+Disallow: /
+
+# Block everything...

lib/yeoman/templates/web/robots.txt

@@ -0,0 +1,15 @@
+User-agent: *
+
+Disallow: /cgi-bin
+Disallow: /feed
+Disallow: /wp/feed
+Disallow: /wp-admin
+Disallow: /wp/wp-admin
+Disallow: /wp-content/cache
+Disallow: /wp/wp-content/cache
+Disallow: /trackback
+Disallow: /wp/trackback
+Disallow: /wp/xmlrpc.php
+Disallow: ?wptheme=
+
+# Sitemap: http://<%= props.domain %>/sitemap.xml

test/functional/08-robots.js

@@ -0,0 +1,18 @@
+'use strict';
+
+var assert  = require('assert');
+var Browser = require('zombie');
+
+describe.only('robots.txt', function(done) {
+  it('on local, should disallow all', function(done) {
+    var browser = new Browser();
+
+    browser
+      .visit('http://local.example.com/robots.txt')
+      .then(function() {
+        assert(!!browser.text().match('# Block everything...'));
+      })
+      .then(done, done)
+    ;
+  });
+});