redo proxy changes

Author: vdusek

src/apify/scrapy/middlewares/apify_proxy.py

@@ -78,11 +78,6 @@ async def process_request(self, request: Request, spider: Spider) -> None:
         Raises:
             ValueError: If username and password are not provided in the proxy URL.
         """
-        # Do not use proxy for robots.txt, as it causes 403 Forbidden.
-        if request.url.endswith('/robots.txt'):
-            request.meta.pop('proxy', None)
-            return
-
         Actor.log.debug(f'ApifyHttpProxyMiddleware.process_request: request={request}, spider={spider}')
         url = await self._get_new_proxy_url()