Force https for urls in scraper (#2222)

* force http to https

* fixing util method to convert to https

* adding test mode field

* adding filters to pull in test_mode resources

* Revert "adding test mode field"

This reverts commit 9af6840.

* Revert "adding filters to pull in test_mode resources"

This reverts commit 2ff2ac9.

Author: shanbady

learning_resources/site_scrapers/utils.py

@@ -5,6 +5,7 @@
 
 
 def scraper_for_site(url):
+    url = url.replace("http://", "https://")
     for pattern in SITE_SCRAPER_MAP:
         if re.search(pattern, url):
             return SITE_SCRAPER_MAP[pattern](url)

learning_resources/site_scrapers/utils_test.py

@@ -29,3 +29,22 @@ def test_scraper_for_site(mocker, url, expected_scraper_class):
 
     scraper = scraper_for_site(url)
     assert isinstance(scraper, expected_scraper_class)
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "http://example.com",
+        "http://micromasters.mit.edu/ds/",
+        "http://unknownsite.com",
+        "http://executive.mit.edu/course/innovation-executive-academy/a05U1000005l8nFIAQ.html",
+    ],
+)
+def test_scraper_forces_https(mocker, url):
+    """
+    Test that the scraper class forces https for the start url
+    """
+
+    scraper = scraper_for_site(url)
+    assert "http://" not in scraper.start_url
+    assert "https://" in scraper.start_url