redo proxy changes

Author: vdusek

src/apify/scrapy/middlewares/apify_proxy.py

@@ -78,11 +78,6 @@ async def process_request(self, request: Request, spider: Spider) -> None:
         Raises:
             ValueError: If username and password are not provided in the proxy URL.
         """
-        # Do not use proxy for robots.txt, as it causes 403 Forbidden.
-        if request.url.endswith('/robots.txt'):
-            request.meta.pop('proxy', None)
-            return
-
         Actor.log.debug(f'ApifyHttpProxyMiddleware.process_request: request={request}, spider={spider}')
         url = await self._get_new_proxy_url()
 
@@ -151,4 +146,7 @@ async def _get_new_proxy_url(self) -> ParseResult:
 
         # Get a new proxy URL and return it
         new_url = await proxy_cfg.new_url()
+
+        print(f'new_url: {new_url}')
+
         return urlparse(str(new_url))