From 27506b7e75927e5dd09761f5eed058580b822771 Mon Sep 17 00:00:00 2001
From: James Moger <james.moger@gitblit.com>
Date: Fri, 15 Jun 2012 08:16:50 -0400
Subject: [PATCH] Configurable robots.txt (issue 99)

---
 distrib/gitblit.properties            |  7 +++
 src/WEB-INF/web.xml                   | 15 +++++-
 src/com/gitblit/RobotsTxtServlet.java | 68 +++++++++++++++++++++++++++
 3 files changed, 89 insertions(+), 1 deletion(-)
 create mode 100644 src/com/gitblit/RobotsTxtServlet.java

diff --git a/distrib/gitblit.properties b/distrib/gitblit.properties
index 440414e7..91b1c2b7 100644
--- a/distrib/gitblit.properties
+++ b/distrib/gitblit.properties
@@ -295,6 +295,13 @@ web.enableRpcManagement = false
 # SINCE 0.7.0 
 web.enableRpcAdministration = false
 
+# Full path to a configurable robots.txt file.  With this file you can control
+# what parts of your Gitblit server respectable robots are allowed to traverse.
+# http://googlewebmastercentral.blogspot.com/2008/06/improving-on-robots-exclusion-protocol.html
+#
+# SINCE 1.0.0
+web.robots.txt = 
+
 # If true, the web ui layout will respond and adapt to the browser's dimensions.
 # if false, the web ui will use a 940px fixed-width layout.
 # http://twitter.github.com/bootstrap/scaffolding.html#responsive
diff --git a/src/WEB-INF/web.xml b/src/WEB-INF/web.xml
index eef49d49..85b24d55 100644
--- a/src/WEB-INF/web.xml
+++ b/src/WEB-INF/web.xml
@@ -99,6 +99,19 @@
 	</servlet-mapping>	
 	
 
+	<!-- Robots.txt Servlet
+		 <url-pattern> MUST match: 
+			* Wicket Filter ignorePaths parameter -->
+	<servlet>
+		<servlet-name>RobotsTxtServlet</servlet-name>
+		<servlet-class>com.gitblit.RobotsTxtServlet</servlet-class>
+	</servlet>
+	<servlet-mapping>
+		<servlet-name>RobotsTxtServlet</servlet-name>		
+		<url-pattern>/robots.txt</url-pattern>
+	</servlet-mapping>
+
+	
 	<!-- Git Access Restriction Filter
 		 <url-pattern> MUST match: 
 			* GitServlet
@@ -202,7 +215,7 @@
              	* PagesFilter <url-pattern>
              	* PagesServlet <url-pattern>
              	* com.gitblit.Constants.PAGES_PATH -->
-            <param-value>git/,feed/,zip/,federation/,rpc/,pages/</param-value>
+            <param-value>git/,feed/,zip/,federation/,rpc/,pages/,robots.txt</param-value>
         </init-param>
     </filter>
     <filter-mapping>
diff --git a/src/com/gitblit/RobotsTxtServlet.java b/src/com/gitblit/RobotsTxtServlet.java
new file mode 100644
index 00000000..c142be0d
--- /dev/null
+++ b/src/com/gitblit/RobotsTxtServlet.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2012 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit;
+
+import java.io.File;
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import com.gitblit.utils.FileUtils;
+import com.gitblit.utils.StringUtils;
+
+/**
+ * Handles requests for robots.txt
+ * 
+ * @author James Moger
+ * 
+ */
+public class RobotsTxtServlet extends HttpServlet {
+
+	private static final long serialVersionUID = 1L;
+
+	public RobotsTxtServlet() {
+		super();
+	}
+	
+	@Override
+	protected void doPost(HttpServletRequest request, HttpServletResponse response)
+			throws ServletException, java.io.IOException {
+		processRequest(request, response);
+	}
+
+	@Override
+	protected void doGet(HttpServletRequest request, HttpServletResponse response)
+			throws ServletException, IOException {
+		processRequest(request, response);
+	}
+
+	protected void processRequest(javax.servlet.http.HttpServletRequest request,
+			javax.servlet.http.HttpServletResponse response) throws javax.servlet.ServletException,
+			java.io.IOException {
+		String robotstxt = GitBlit.getString(Keys.web.robots.txt, null);		
+		String content = "";
+		if (!StringUtils.isEmpty(robotstxt)) {
+			File robotsfile = new File(robotstxt);
+			if (robotsfile.exists()) {
+				content = FileUtils.readContent(robotsfile, "\n");
+			}
+		}
+		response.getWriter().append(content);
+	}
+}
-- 
2.39.5