⏫ Forwardport of #12310 to 2.3-develop branch

Applied pull request patch https://github.com/magento/magento2/pull/12310.patch (created by @tufahu) based on commit(s):
  1. 8fc99b1
  2. 36dfdca
  3. 5c02402
  4. 2eada9c
  5. e9986c5

Author: magento-engcom-team

app/code/Magento/Robots/Controller/Index/Index.php

@@ -43,6 +43,7 @@ public function execute()
         /** @var Page $resultPage */
         $resultPage = $this->resultPageFactory->create(true);
         $resultPage->addHandle('robots_index_index');
+        $resultPage->setHeader('Content-Type', 'text/plain');
         return $resultPage;
     }
 }

app/code/Magento/Robots/Test/Unit/Controller/Index/IndexTest.php

@@ -51,6 +51,9 @@ public function testExecute()
         $resultPageMock->expects($this->once())
             ->method('addHandle')
             ->with('robots_index_index');
+        $resultPageMock->expects($this->once())
+            ->method('setHeader')
+            ->with('Content-Type', 'text/plain');
 
         $this->resultPageFactory->expects($this->any())
             ->method('create')