Change X-Robots-Tag header from "none" to "noindex, nofollow"

While "none" is indeed equivalent to "noindex, nofollow" for Google, but seems to be not supported by Bing and probably other search engines.

https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta/name#other_metadata_names
https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag?hl=de#comma-separated-list
https://www.bing.com/webmasters/help/which-robots-metatags-does-bing-support-5198d240

Signed-off-by: MichaIng <micha@dietpi.com>

.htaccess

@@ -31,7 +31,7 @@
     Header always set X-Permitted-Cross-Domain-Policies "none"
 
     Header onsuccess unset X-Robots-Tag
-    Header always set X-Robots-Tag "none"
+    Header always set X-Robots-Tag "noindex, nofollow"
 
     Header onsuccess unset X-XSS-Protection
     Header always set X-XSS-Protection "1; mode=block"

build/integration/features/carddav.feature

@@ -46,7 +46,7 @@ Feature: carddav
         |X-Content-Type-Options |nosniff|
         |X-Frame-Options|SAMEORIGIN|
         |X-Permitted-Cross-Domain-Policies|none|
-        |X-Robots-Tag|none|
+        |X-Robots-Tag|noindex, nofollow|
         |X-XSS-Protection|1; mode=block|
 
   Scenario: Exporting the picture of ones own contact
@@ -60,5 +60,5 @@ Feature: carddav
       |X-Content-Type-Options |nosniff|
       |X-Frame-Options|SAMEORIGIN|
       |X-Permitted-Cross-Domain-Policies|none|
-      |X-Robots-Tag|none|
+      |X-Robots-Tag|noindex, nofollow|
       |X-XSS-Protection|1; mode=block|

build/integration/features/dav-v2.feature

@@ -27,7 +27,7 @@ Feature: dav-v2
 			|X-Content-Type-Options |nosniff|
 			|X-Frame-Options|SAMEORIGIN|
 			|X-Permitted-Cross-Domain-Policies|none|
-			|X-Robots-Tag|none|
+			|X-Robots-Tag|noindex, nofollow|
 			|X-XSS-Protection|1; mode=block|
 		And Downloaded content should start with "Welcome to your Nextcloud account!"
 

build/integration/features/webdav-related.feature

@@ -53,7 +53,7 @@ Feature: webdav-related
 		And User "user0" moves file "/textfile0.txt" to "/testshare/textfile0.txt"
 		And the HTTP status code should be "403"
 		When Downloading file "/testshare/textfile0.txt"
- 		Then the HTTP status code should be "404"
+		Then the HTTP status code should be "404"
 
 	Scenario: Moving a file to overwrite a file in a folder with no permissions
 		Given using old dav path
@@ -251,7 +251,7 @@ Feature: webdav-related
 			|X-Content-Type-Options |nosniff|
 			|X-Frame-Options|SAMEORIGIN|
 			|X-Permitted-Cross-Domain-Policies|none|
-			|X-Robots-Tag|none|
+			|X-Robots-Tag|noindex, nofollow|
 			|X-XSS-Protection|1; mode=block|
 		And Downloaded content should start with "Welcome to your Nextcloud account!"
 

core/js/setupchecks.js

@@ -628,13 +628,13 @@
 			if (xhr.status === 200) {
 				var securityHeaders = {
 					'X-Content-Type-Options': ['nosniff'],
-					'X-Robots-Tag': ['none'],
+					'X-Robots-Tag': ['noindex, nofollow'],
 					'X-Frame-Options': ['SAMEORIGIN', 'DENY'],
 					'X-Permitted-Cross-Domain-Policies': ['none'],
 				};
 				for (var header in securityHeaders) {
 					var option = securityHeaders[header][0];
-					if(!xhr.getResponseHeader(header) || xhr.getResponseHeader(header).toLowerCase() !== option.toLowerCase()) {
+					if(!xhr.getResponseHeader(header) || xhr.getResponseHeader(header).replace(/, /, ',').toLowerCase() !== option.replace(/, /, ',').toLowerCase()) {
 						var msg = t('core', 'The "{header}" HTTP header is not set to "{expected}". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.', {header: header, expected: option});
 						if(xhr.getResponseHeader(header) && securityHeaders[header].length > 1 && xhr.getResponseHeader(header).toLowerCase() === securityHeaders[header][1].toLowerCase()) {
 							msg = t('core', 'The "{header}" HTTP header is not set to "{expected}". Some features might not work correctly, as it is recommended to adjust this setting accordingly.', {header: header, expected: option});

core/js/tests/specs/setupchecksSpec.js

@@ -1569,7 +1569,7 @@ describe('OC.SetupChecks tests', function() {
 					msg: 'The "X-Content-Type-Options" HTTP header is not set to "nosniff". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
 					type: OC.SetupChecks.MESSAGE_TYPE_WARNING
 				}, {
-					msg: 'The "X-Robots-Tag" HTTP header is not set to "none". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
+					msg: 'The "X-Robots-Tag" HTTP header is not set to "noindex, nofollow". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
 					type: OC.SetupChecks.MESSAGE_TYPE_WARNING
 				}, {
 					msg: 'The "X-Frame-Options" HTTP header is not set to "SAMEORIGIN". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
@@ -1596,7 +1596,7 @@ describe('OC.SetupChecks tests', function() {
 			suite.server.requests[0].respond(
 				200,
 				{
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'Strict-Transport-Security': 'max-age=15768000;preload',
 					'X-Permitted-Cross-Domain-Policies': 'none',
@@ -1627,7 +1627,7 @@ describe('OC.SetupChecks tests', function() {
 				{
 					'X-XSS-Protection': '1; mode=block',
 					'X-Content-Type-Options': 'nosniff',
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'Strict-Transport-Security': 'max-age=15768000',
 					'X-Permitted-Cross-Domain-Policies': 'none',
@@ -1641,6 +1641,49 @@ describe('OC.SetupChecks tests', function() {
 			});
 		});
 
+		describe('check X-Robots-Tag header', function() {
+			it('should return no message if X-Robots-Tag is set to noindex,nofollow without space', function(done) {
+				protocolStub.returns('https');
+				var result = OC.SetupChecks.checkGeneric();
+				suite.server.requests[0].respond(200, {
+					'Strict-Transport-Security': 'max-age=15768000',
+					'X-XSS-Protection': '1; mode=block',
+					'X-Content-Type-Options': 'nosniff',
+					'X-Robots-Tag': 'noindex,nofollow',
+					'X-Frame-Options': 'SAMEORIGIN',
+					'X-Permitted-Cross-Domain-Policies': 'none',
+					'Referrer-Policy': 'no-referrer',
+				});
+				result.done(function( data, s, x ){
+					expect(data).toEqual([]);
+					done();
+				});
+			});
+
+			it('should return a message if X-Robots-Tag is set to none', function(done) {
+				protocolStub.returns('https');
+				var result = OC.SetupChecks.checkGeneric();
+				suite.server.requests[0].respond(200, {
+					'Strict-Transport-Security': 'max-age=15768000',
+					'X-XSS-Protection': '1; mode=block',
+					'X-Content-Type-Options': 'nosniff',
+					'X-Robots-Tag': 'none',
+					'X-Frame-Options': 'SAMEORIGIN',
+					'X-Permitted-Cross-Domain-Policies': 'none',
+					'Referrer-Policy': 'no-referrer',
+				});
+				result.done(function( data, s, x ){
+					expect(data).toEqual([
+						{
+							msg: 'The "X-Robots-Tag" HTTP header is not set to "noindex, nofollow". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
+							type: OC.SetupChecks.MESSAGE_TYPE_WARNING
+						}
+					]);
+					done();
+				});
+			});
+		});
+
 		describe('check X-XSS-Protection header', function() {
 			it('should return no message if X-XSS-Protection is set to 1; mode=block; report=https://example.com', function(done) {
 				protocolStub.returns('https');
@@ -1650,7 +1693,7 @@ describe('OC.SetupChecks tests', function() {
 					'Strict-Transport-Security': 'max-age=15768000',
 					'X-XSS-Protection': '1; mode=block; report=https://example.com',
 					'X-Content-Type-Options': 'nosniff',
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'X-Permitted-Cross-Domain-Policies': 'none',
 					'Referrer-Policy': 'no-referrer',
@@ -1670,7 +1713,7 @@ describe('OC.SetupChecks tests', function() {
 					'Strict-Transport-Security': 'max-age=15768000',
 					'X-XSS-Protection': '1; mode=block',
 					'X-Content-Type-Options': 'nosniff',
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'X-Permitted-Cross-Domain-Policies': 'none',
 					'Referrer-Policy': 'no-referrer',
@@ -1690,7 +1733,7 @@ describe('OC.SetupChecks tests', function() {
 					'Strict-Transport-Security': 'max-age=15768000',
 					'X-XSS-Protection': '1',
 					'X-Content-Type-Options': 'nosniff',
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'X-Permitted-Cross-Domain-Policies': 'none',
 					'Referrer-Policy': 'no-referrer',
@@ -1715,7 +1758,7 @@ describe('OC.SetupChecks tests', function() {
 					'Strict-Transport-Security': 'max-age=15768000',
 					'X-XSS-Protection': '0',
 					'X-Content-Type-Options': 'nosniff',
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'X-Permitted-Cross-Domain-Policies': 'none',
 					'Referrer-Policy': 'no-referrer',
@@ -1742,7 +1785,7 @@ describe('OC.SetupChecks tests', function() {
 					'Strict-Transport-Security': 'max-age=15768000',
 					'X-XSS-Protection': '1; mode=block',
 					'X-Content-Type-Options': 'nosniff',
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'X-Permitted-Cross-Domain-Policies': 'none',
 					'Referrer-Policy': 'no-referrer',
@@ -1762,7 +1805,7 @@ describe('OC.SetupChecks tests', function() {
 					'Strict-Transport-Security': 'max-age=15768000',
 					'X-XSS-Protection': '1; mode=block',
 					'X-Content-Type-Options': 'nosniff',
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'X-Permitted-Cross-Domain-Policies': 'none',
 					'Referrer-Policy': 'no-referrer-when-downgrade',
@@ -1782,7 +1825,7 @@ describe('OC.SetupChecks tests', function() {
 					'Strict-Transport-Security': 'max-age=15768000',
 					'X-XSS-Protection': '1; mode=block',
 					'X-Content-Type-Options': 'nosniff',
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'X-Permitted-Cross-Domain-Policies': 'none',
 					'Referrer-Policy': 'strict-origin',
@@ -1802,7 +1845,7 @@ describe('OC.SetupChecks tests', function() {
 					'Strict-Transport-Security': 'max-age=15768000',
 					'X-XSS-Protection': '1; mode=block',
 					'X-Content-Type-Options': 'nosniff',
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'X-Permitted-Cross-Domain-Policies': 'none',
 					'Referrer-Policy': 'strict-origin-when-cross-origin',
@@ -1822,7 +1865,7 @@ describe('OC.SetupChecks tests', function() {
 					'Strict-Transport-Security': 'max-age=15768000',
 					'X-XSS-Protection': '1; mode=block',
 					'X-Content-Type-Options': 'nosniff',
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'X-Permitted-Cross-Domain-Policies': 'none',
 					'Referrer-Policy': 'same-origin',
@@ -1842,7 +1885,7 @@ describe('OC.SetupChecks tests', function() {
 					'Strict-Transport-Security': 'max-age=15768000',
 					'X-XSS-Protection': '1; mode=block',
 					'X-Content-Type-Options': 'nosniff',
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'X-Permitted-Cross-Domain-Policies': 'none',
 					'Referrer-Policy': 'origin',
@@ -1867,7 +1910,7 @@ describe('OC.SetupChecks tests', function() {
 					'Strict-Transport-Security': 'max-age=15768000',
 					'X-XSS-Protection': '1; mode=block',
 					'X-Content-Type-Options': 'nosniff',
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'X-Permitted-Cross-Domain-Policies': 'none',
 					'Referrer-Policy': 'origin-when-cross-origin',
@@ -1892,7 +1935,7 @@ describe('OC.SetupChecks tests', function() {
 					'Strict-Transport-Security': 'max-age=15768000',
 					'X-XSS-Protection': '1; mode=block',
 					'X-Content-Type-Options': 'nosniff',
-					'X-Robots-Tag': 'none',
+					'X-Robots-Tag': 'noindex, nofollow',
 					'X-Frame-Options': 'SAMEORIGIN',
 					'X-Permitted-Cross-Domain-Policies': 'none',
 					'Referrer-Policy': 'unsafe-url',
@@ -1919,7 +1962,7 @@ describe('OC.SetupChecks tests', function() {
 			{
 				'X-XSS-Protection': '1; mode=block',
 				'X-Content-Type-Options': 'nosniff',
-				'X-Robots-Tag': 'none',
+				'X-Robots-Tag': 'noindex, nofollow',
 				'X-Frame-Options': 'SAMEORIGIN',
 				'X-Permitted-Cross-Domain-Policies': 'none',
 				'Referrer-Policy': 'no-referrer',
@@ -1965,7 +2008,7 @@ describe('OC.SetupChecks tests', function() {
 			{
 				'X-XSS-Protection': '1; mode=block',
 				'X-Content-Type-Options': 'nosniff',
-				'X-Robots-Tag': 'none',
+				'X-Robots-Tag': 'noindex, nofollow',
 				'X-Frame-Options': 'SAMEORIGIN',
 				'X-Permitted-Cross-Domain-Policies': 'none',
 				'Referrer-Policy': 'no-referrer',
@@ -1990,7 +2033,7 @@ describe('OC.SetupChecks tests', function() {
 				'Strict-Transport-Security': 'max-age=15551999',
 				'X-XSS-Protection': '1; mode=block',
 				'X-Content-Type-Options': 'nosniff',
-				'X-Robots-Tag': 'none',
+				'X-Robots-Tag': 'noindex, nofollow',
 				'X-Frame-Options': 'SAMEORIGIN',
 				'X-Permitted-Cross-Domain-Policies': 'none',
 				'Referrer-Policy': 'no-referrer',
@@ -2015,7 +2058,7 @@ describe('OC.SetupChecks tests', function() {
 				'Strict-Transport-Security': 'iAmABogusHeader342',
 				'X-XSS-Protection': '1; mode=block',
 				'X-Content-Type-Options': 'nosniff',
-				'X-Robots-Tag': 'none',
+				'X-Robots-Tag': 'noindex, nofollow',
 				'X-Frame-Options': 'SAMEORIGIN',
 				'X-Permitted-Cross-Domain-Policies': 'none',
 				'Referrer-Policy': 'no-referrer',
@@ -2039,7 +2082,7 @@ describe('OC.SetupChecks tests', function() {
 			'Strict-Transport-Security': 'max-age=15768000',
 			'X-XSS-Protection': '1; mode=block',
 			'X-Content-Type-Options': 'nosniff',
-			'X-Robots-Tag': 'none',
+			'X-Robots-Tag': 'noindex, nofollow',
 			'X-Frame-Options': 'SAMEORIGIN',
 			'X-Permitted-Cross-Domain-Policies': 'none',
 			'Referrer-Policy': 'no-referrer',
@@ -2059,7 +2102,7 @@ describe('OC.SetupChecks tests', function() {
 			'Strict-Transport-Security': 'max-age=99999999',
 			'X-XSS-Protection': '1; mode=block',
 			'X-Content-Type-Options': 'nosniff',
-			'X-Robots-Tag': 'none',
+			'X-Robots-Tag': 'noindex, nofollow',
 			'X-Frame-Options': 'SAMEORIGIN',
 			'X-Permitted-Cross-Domain-Policies': 'none',
 			'Referrer-Policy': 'no-referrer',
@@ -2079,7 +2122,7 @@ describe('OC.SetupChecks tests', function() {
 			'Strict-Transport-Security': 'max-age=99999999; includeSubDomains',
 			'X-XSS-Protection': '1; mode=block',
 			'X-Content-Type-Options': 'nosniff',
-			'X-Robots-Tag': 'none',
+			'X-Robots-Tag': 'noindex, nofollow',
 			'X-Frame-Options': 'SAMEORIGIN',
 			'X-Permitted-Cross-Domain-Policies': 'none',
 			'Referrer-Policy': 'no-referrer',
@@ -2099,7 +2142,7 @@ describe('OC.SetupChecks tests', function() {
 			'Strict-Transport-Security': 'max-age=99999999; preload; includeSubDomains',
 			'X-XSS-Protection': '1; mode=block',
 			'X-Content-Type-Options': 'nosniff',
-			'X-Robots-Tag': 'none',
+			'X-Robots-Tag': 'noindex, nofollow',
 			'X-Frame-Options': 'SAMEORIGIN',
 			'X-Permitted-Cross-Domain-Policies': 'none',
 			'Referrer-Policy': 'no-referrer',

lib/private/legacy/OC_Response.php

@@ -99,7 +99,7 @@ class OC_Response {
 			header('X-Content-Type-Options: nosniff'); // Disable sniffing the content type for IE
 			header('X-Frame-Options: SAMEORIGIN'); // Disallow iFraming from other domains
 			header('X-Permitted-Cross-Domain-Policies: none'); // https://www.adobe.com/devnet/adobe-media-server/articles/cross-domain-xml-for-streaming.html
-			header('X-Robots-Tag: none'); // https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag
+			header('X-Robots-Tag: noindex, nofollow'); // https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag
 			header('X-XSS-Protection: 1; mode=block'); // Enforce browser based XSS filters
 		}
 	}

lib/public/AppFramework/Http/Response.php

@@ -257,7 +257,7 @@ class Response {
 
 		$this->headers['Content-Security-Policy'] = $this->getContentSecurityPolicy()->buildPolicy();
 		$this->headers['Feature-Policy'] = $this->getFeaturePolicy()->buildPolicy();
-		$this->headers['X-Robots-Tag'] = 'none';
+		$this->headers['X-Robots-Tag'] = 'noindex, nofollow';
 
 		if ($this->ETag) {
 			$mergeWith['ETag'] = '"' . $this->ETag . '"';

tests/data/setUploadLimit/htaccess

@@ -11,7 +11,7 @@
     # Add security and privacy related headers
     Header set X-Content-Type-Options "nosniff"
     Header set X-XSS-Protection "1; mode=block"
-    Header set X-Robots-Tag "none"
+    Header set X-Robots-Tag "noindex, nofollow"
     Header set X-Frame-Options "SAMEORIGIN"
     SetEnv modHeadersAvailable true
   </IfModule>

tests/lib/AppFramework/Controller/ControllerTest.php

@@ -116,7 +116,7 @@ class ControllerTest extends \Test\TestCase {
 			'Content-Security-Policy' => "default-src 'none';base-uri 'none';manifest-src 'self';frame-ancestors 'none'",
 			'Feature-Policy' => "autoplay 'none';camera 'none';fullscreen 'none';geolocation 'none';microphone 'none';payment 'none'",
 			'X-Request-Id' => $this->request->getId(),
-			'X-Robots-Tag' => 'none',
+			'X-Robots-Tag' => 'noindex, nofollow',
 		];
 
 		$response = $this->controller->customDataResponse(['hi']);

tests/lib/AppFramework/Http/DataResponseTest.php

@@ -67,7 +67,7 @@ class DataResponseTest extends \Test\TestCase {
 			'Cache-Control' => 'no-cache, no-store, must-revalidate',
 			'Content-Security-Policy' => "default-src 'none';base-uri 'none';manifest-src 'self';frame-ancestors 'none'",
 			'Feature-Policy' => "autoplay 'none';camera 'none';fullscreen 'none';geolocation 'none';microphone 'none';payment 'none'",
-			'X-Robots-Tag' => 'none',
+			'X-Robots-Tag' => 'noindex, nofollow',
 			'X-Request-Id' => \OC::$server->get(IRequest::class)->getId(),
 		];
 		$expectedHeaders = array_merge($expectedHeaders, $headers);

tests/lib/AppFramework/Http/ResponseTest.php

@@ -51,7 +51,7 @@ class ResponseTest extends \Test\TestCase {
 			'Last-Modified' => 1,
 			'ETag' => 3,
 			'Something-Else' => 'hi',
-			'X-Robots-Tag' => 'none',
+			'X-Robots-Tag' => 'noindex, nofollow',
 		];
 
 		$this->childResponse->setHeaders($expected);