change back return proxy from spider to httpclientdownloader #128

Author: code4craft

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -2,6 +2,7 @@
 
 import com.google.common.collect.Lists;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.http.HttpHost;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import us.codecraft.webmagic.downloader.Downloader;
@@ -324,6 +325,10 @@ public void run() {
                             onError(requestFinal);
                             logger.error("process request " + requestFinal + " error", e);
                         } finally {
+                            if (site.getHttpProxyPool().isEnable()) {
+                                site.returnHttpProxyToPool((HttpHost) requestFinal.getExtra(Request.PROXY), (Integer) requestFinal
+                                        .getExtra(Request.STATUS_CODE));
+                            }
                             pageCount.incrementAndGet();
                             signalNewUrl();
                         }

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java

@@ -117,10 +117,6 @@ public Page download(Request request, Task task) {
             } catch (IOException e) {
                 logger.warn("close response fail", e);
             }
-            if (site.getHttpProxyPool().isEnable()) {
-                site.returnHttpProxyToPool((HttpHost) request.getExtra(Request.PROXY), (Integer) request
-                        .getExtra(Request.STATUS_CODE));
-            }
         }
     }