Merge pull request #361 from duckontheweb/windows-ci-fixes

Fix Windows CI builds

Author: Jon Duckworth

.github/workflows/continuous-integration.yml

@@ -7,15 +7,22 @@ on:
   pull_request:
 
 jobs:
-  build:
-    name: build
+  test:
+    name: test
     runs-on: ${{ matrix.os }}
     strategy:
       # Allow other matrix jobs to complete if 1 fails
       fail-fast: false
       matrix:
-        python-version: ["3.6", "3.7", "3.8"]
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version:
+          - "3.6"
+          - "3.7"
+          - "3.8"
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
+
     steps:
       - uses: actions/checkout@v2
 
@@ -24,15 +31,38 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Cache dependencies
+      - name: Cache dependencies (Linux)
+        if: startsWith(runner.os, 'Linux')
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
           # Cache based on OS, Python version, and dependency hash
-          key: pip-${{ runner.os }}-python${{ matrix.python-version }}-${{ hashFiles('requirements-dev.txt') }}
+          key: test-${{ runner.os }}-python${{ matrix.python-version }}-${{ hashFiles('requirements-test.txt') }}
+      
+      - name: Cache dependencies (macOS)
+        if: startsWith(runner.os, 'macOS')
+        uses: actions/cache@v2
+        with:
+          path: ~/Library/Caches/pip
+          # Cache based on OS, Python version, and dependency hash
+          key: test-${{ runner.os }}-python${{ matrix.python-version }}-${{ hashFiles('requirements-test.txt') }}
+          
+      - name: Cache dependencies (Windows)
+        if: startsWith(runner.os, 'Windows')
+        uses: actions/cache@v2
+        with:
+          path: ~\AppData\Local\pip\Cache
+          # Cache based on OS, Python version, and dependency hash
+          key: pip-${{ runner.os }}-python${{ matrix.python-version }}-${{ hashFiles('requirements-test.txt') }}
+      
+      - name: Install dependencies
+        run: |
+          pip install -r requirements-test.txt
+          pip install -e ".[validation]"
 
-      - name: Execute linters and test suites
-        run: ./scripts/cibuild
+      - name: Execute test suite
+        run: ./scripts/test
+        shell: bash
 
       - name: Upload All coverage to Codecov
         uses: codecov/codecov-action@v1
@@ -42,14 +72,50 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ./coverage.xml
           fail_ci_if_error: false 
-  vanilla:
+  lint:
     runs-on: ubuntu-latest
+    strategy:
+      # Allow other matrix jobs to complete if 1 fails
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.6"
+          - "3.7"
+          - "3.8"
+
     steps:
       - uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Cache dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          # Cache based on OS, Python version, and dependency hash
+          key: lint-${{ runner.os }}-python${{ matrix.python-version }}-${{ hashFiles('requirements-test.txt') }}
+      
+      - name: Install dependencies
+        run: |
+          pip install -r requirements-test.txt
+
+      - name: Execute linters & type checkers
+        run: ./scripts/lint
+
+  vanilla:
+    runs-on: ubuntu-latest
+    steps:  
+      - uses: actions/checkout@v2
+      
       - uses: actions/setup-python@v2
         with:
           python-version: "3.8"
+
       - name: Install without orjson
         run: pip install '.[validation]'
+      
       - name: Run unittests
         run: python -m unittest discover tests

.gitignore

@@ -1,3 +1,4 @@
+tmp*
 *.pyc
 *.egg-info
 build/

pystac/layout.py

@@ -1,10 +1,10 @@
 from abc import abstractmethod, ABC
 from collections import OrderedDict
-import os
 from string import Formatter
 from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING, Union
 
 import pystac
+from pystac.utils import safe_urlparse, join_path_or_url, JoinType
 
 if TYPE_CHECKING:
     from pystac.stac_object import STACObject as STACObject_Type
@@ -374,36 +374,51 @@ def __init__(
     def get_catalog_href(
         self, cat: "Catalog_Type", parent_dir: str, is_root: bool
     ) -> str:
+        parsed_parent_dir = safe_urlparse(parent_dir)
+        join_type = JoinType.from_parsed_uri(parsed_parent_dir)
+
         if is_root or self.catalog_template is None:
             return self.fallback_strategy.get_catalog_href(cat, parent_dir, is_root)
         else:
             template_path = self.catalog_template.substitute(cat)
             if not template_path.endswith(".json"):
-                template_path = os.path.join(template_path, cat.DEFAULT_FILE_NAME)
+                template_path = join_path_or_url(
+                    join_type, template_path, cat.DEFAULT_FILE_NAME
+                )
 
-            return os.path.join(parent_dir, template_path)
+            return join_path_or_url(join_type, parent_dir, template_path)
 
     def get_collection_href(
         self, col: "Collection_Type", parent_dir: str, is_root: bool
     ) -> str:
+        parsed_parent_dir = safe_urlparse(parent_dir)
+        join_type = JoinType.from_parsed_uri(parsed_parent_dir)
+
         if is_root or self.collection_template is None:
             return self.fallback_strategy.get_collection_href(col, parent_dir, is_root)
         else:
             template_path = self.collection_template.substitute(col)
             if not template_path.endswith(".json"):
-                template_path = os.path.join(template_path, col.DEFAULT_FILE_NAME)
+                template_path = join_path_or_url(
+                    join_type, template_path, col.DEFAULT_FILE_NAME
+                )
 
-            return os.path.join(parent_dir, template_path)
+            return join_path_or_url(join_type, parent_dir, template_path)
 
     def get_item_href(self, item: "Item_Type", parent_dir: str) -> str:
+        parsed_parent_dir = safe_urlparse(parent_dir)
+        join_type = JoinType.from_parsed_uri(parsed_parent_dir)
+
         if self.item_template is None:
             return self.fallback_strategy.get_item_href(item, parent_dir)
         else:
             template_path = self.item_template.substitute(item)
             if not template_path.endswith(".json"):
-                template_path = os.path.join(template_path, "{}.json".format(item.id))
+                template_path = join_path_or_url(
+                    join_type, template_path, "{}.json".format(item.id)
+                )
 
-            return os.path.join(parent_dir, template_path)
+            return join_path_or_url(join_type, parent_dir, template_path)
 
 
 class BestPracticesLayoutStrategy(HrefLayoutStrategy):
@@ -424,24 +439,33 @@ class BestPracticesLayoutStrategy(HrefLayoutStrategy):
     def get_catalog_href(
         self, cat: "Catalog_Type", parent_dir: str, is_root: bool
     ) -> str:
+        parsed_parent_dir = safe_urlparse(parent_dir)
+        join_type = JoinType.from_parsed_uri(parsed_parent_dir)
+
         if is_root:
             cat_root = parent_dir
         else:
-            cat_root = os.path.join(parent_dir, "{}".format(cat.id))
+            cat_root = join_path_or_url(join_type, parent_dir, "{}".format(cat.id))
 
-        return os.path.join(cat_root, cat.DEFAULT_FILE_NAME)
+        return join_path_or_url(join_type, cat_root, cat.DEFAULT_FILE_NAME)
 
     def get_collection_href(
         self, col: "Collection_Type", parent_dir: str, is_root: bool
     ) -> str:
+        parsed_parent_dir = safe_urlparse(parent_dir)
+        join_type = JoinType.from_parsed_uri(parsed_parent_dir)
+
         if is_root:
             col_root = parent_dir
         else:
-            col_root = os.path.join(parent_dir, "{}".format(col.id))
+            col_root = join_path_or_url(join_type, parent_dir, "{}".format(col.id))
 
-        return os.path.join(col_root, col.DEFAULT_FILE_NAME)
+        return join_path_or_url(join_type, col_root, col.DEFAULT_FILE_NAME)
 
     def get_item_href(self, item: "Item_Type", parent_dir: str) -> str:
-        item_root = os.path.join(parent_dir, "{}".format(item.id))
+        parsed_parent_dir = safe_urlparse(parent_dir)
+        join_type = JoinType.from_parsed_uri(parsed_parent_dir)
+
+        item_root = join_path_or_url(join_type, parent_dir, "{}".format(item.id))
 
-        return os.path.join(item_root, "{}.json".format(item.id))
+        return join_path_or_url(join_type, item_root, "{}.json".format(item.id))

pystac/stac_io.py

@@ -14,11 +14,11 @@
 )
 import warnings
 
-from urllib.parse import urlparse
 from urllib.request import urlopen
 from urllib.error import HTTPError
 
 import pystac
+from pystac.utils import safe_urlparse
 import pystac.serialization
 
 # Use orjson if available
@@ -182,7 +182,7 @@ def read_text(
         return self.read_text_from_href(href, *args, **kwargs)
 
     def read_text_from_href(self, href: str, *args: Any, **kwargs: Any) -> str:
-        parsed = urlparse(href)
+        parsed = safe_urlparse(href)
         href_contents: str
         if parsed.scheme != "":
             try:
@@ -191,9 +191,8 @@ def read_text_from_href(self, href: str, *args: Any, **kwargs: Any) -> str:
             except HTTPError as e:
                 raise Exception("Could not read uri {}".format(href)) from e
         else:
-            with open(href) as f:
+            with open(href, encoding="utf-8") as f:
                 href_contents = f.read()
-
         return href_contents
 
     def write_text(
@@ -214,7 +213,7 @@ def write_text_to_href(
         dirname = os.path.dirname(href)
         if dirname != "" and not os.path.isdir(dirname):
             os.makedirs(dirname)
-        with open(href, "w") as f:
+        with open(href, "w", encoding="utf-8") as f:
             f.write(txt)