From 6fab234e53b9947f975df4825d222a3b2d25c0d2 Mon Sep 17 00:00:00 2001
From: Evan Kaufman <evan@digitalflophouse.com>
Date: Fri, 1 May 2015 10:14:03 -0500
Subject: [PATCH 1/2] Added robots.txt for canonical url, and no_robots.txt for
 non canonical urls

Pulled htaccess templating out of #37

Based on genesis/wordpress robots: https://github.com/genesis/wordpress/blob/1e4ef9a7d72c612aabe96d8aa8d9da8a35b49610/generator/app/templates/web/robots.txt
---
 lib/yeoman/index.js                    | 12 ++++++++++++
 lib/yeoman/templates/web/_htaccess     | 14 ++++++++++++--
 lib/yeoman/templates/web/no_robots.txt |  5 +++++
 lib/yeoman/templates/web/robots.txt    | 15 +++++++++++++++
 4 files changed, 44 insertions(+), 2 deletions(-)
 create mode 100644 lib/yeoman/templates/web/no_robots.txt
 create mode 100644 lib/yeoman/templates/web/robots.txt

diff --git a/lib/yeoman/index.js b/lib/yeoman/index.js
index 3626cd96..a76b1453 100644
--- a/lib/yeoman/index.js
+++ b/lib/yeoman/index.js
@@ -409,6 +409,18 @@ var WordpressGenerator = yeoman.generators.Base.extend({
     }
   },
 
+  prepareHtaccess: function() {
+    try {
+      this.htaccessFile = this.readFileAsString(path.join(this.env.cwd, 'web', '.htaccess'));
+      this.htaccessFile = this.htaccessFile.replace(/# BEGIN Evolution WordPress(?:.|[\r\n]+)+?# END Evolution WordPress[\r\n]*/i, '');
+
+      this.htaccessWpBlock = !!this.htaccessFile.match(/# BEGIN WordPress(?:.|[\r\n]+)+?# END WordPress[\r\n]*/i);
+    } catch(e) {
+      this.htaccessFile = '';
+      this.htaccessWpBlock = false;
+    }
+  },
+
   prepareRoles: function() {
     this.props.roles = WordpressGenerator.roles.map(function(role) {
       role.checked = !!~this.props.roles.indexOf(role.value);
diff --git a/lib/yeoman/templates/web/_htaccess b/lib/yeoman/templates/web/_htaccess
index 77e1e877..f9717f37 100644
--- a/lib/yeoman/templates/web/_htaccess
+++ b/lib/yeoman/templates/web/_htaccess
@@ -1,4 +1,14 @@
-# BEGIN WordPress
+# BEGIN Evolution WordPress
+<IfModule mod_rewrite.c>
+RewriteEngine On
+RewriteBase /
+# Prevent spidering of non-canonical URLs, such as `[*.]staging.*` and `production.*`.
+RewriteCond %{HTTP_HOST} (^local|staging|^production)\.<%= props.domain.replace('.', '\\.') %>$ [NC]
+RewriteRule ^robots.txt$ no_robots.txt [L]
+</IfModule>
+# END Evolution WordPress
+<%= htaccessFile %>
+<% if (!htaccessWpBlock) { %># BEGIN WordPress
 <IfModule mod_rewrite.c>
 RewriteEngine On
 RewriteBase /
@@ -7,4 +17,4 @@ RewriteCond %{REQUEST_FILENAME} !-f
 RewriteCond %{REQUEST_FILENAME} !-d
 RewriteRule . /index.php [L]
 </IfModule>
-# END WordPress
+# END WordPress<% } %>
diff --git a/lib/yeoman/templates/web/no_robots.txt b/lib/yeoman/templates/web/no_robots.txt
new file mode 100644
index 00000000..2ec669bc
--- /dev/null
+++ b/lib/yeoman/templates/web/no_robots.txt
@@ -0,0 +1,5 @@
+User-agent: *
+
+Disallow: /
+
+# Block everything...
diff --git a/lib/yeoman/templates/web/robots.txt b/lib/yeoman/templates/web/robots.txt
new file mode 100644
index 00000000..03275bf8
--- /dev/null
+++ b/lib/yeoman/templates/web/robots.txt
@@ -0,0 +1,15 @@
+User-agent: *
+
+Disallow: /cgi-bin
+Disallow: /feed
+Disallow: /wp/feed
+Disallow: /wp-admin
+Disallow: /wp/wp-admin
+Disallow: /wp-content/cache
+Disallow: /wp/wp-content/cache
+Disallow: /trackback
+Disallow: /wp/trackback
+Disallow: /wp/xmlrpc.php
+Disallow: ?wptheme=
+
+# Sitemap: http://<%= props.domain %>/sitemap.xml

From 48f5cd48f8a6e4ce94f77cd6de6632a758224c30 Mon Sep 17 00:00:00 2001
From: Evan Kaufman <evan@digitalflophouse.com>
Date: Fri, 1 May 2015 11:37:45 -0500
Subject: [PATCH 2/2] Added test

---
 test/functional/{wordpress => }/07-sni.js |  0
 test/functional/08-robots.js              | 18 ++++++++++++++++++
 2 files changed, 18 insertions(+)
 rename test/functional/{wordpress => }/07-sni.js (100%)
 create mode 100644 test/functional/08-robots.js

diff --git a/test/functional/wordpress/07-sni.js b/test/functional/07-sni.js
similarity index 100%
rename from test/functional/wordpress/07-sni.js
rename to test/functional/07-sni.js
diff --git a/test/functional/08-robots.js b/test/functional/08-robots.js
new file mode 100644
index 00000000..f5422230
--- /dev/null
+++ b/test/functional/08-robots.js
@@ -0,0 +1,18 @@
+'use strict';
+
+var assert  = require('assert');
+var Browser = require('zombie');
+
+describe.only('robots.txt', function(done) {
+  it('on local, should disallow all', function(done) {
+    var browser = new Browser();
+
+    browser
+      .visit('http://local.example.com/robots.txt')
+      .then(function() {
+        assert(!!browser.text().match('# Block everything...'));
+      })
+      .then(done, done)
+    ;
+  });
+});