Add all IANA assigned HTTP method names (#751)

* Ignore `docs/_build`

* Remove `v` prefix from VERSION.  Also added a `lib-doc` target

* Raise a `ValueError` instead of `NotImplementedError`

* Add all registered http method verbs

* Generate `_scm_version.py` on every `make` invocation.  Fix `v` prefix bug.

* `+proxy` for emails

* Make explicit that this script writes to file

* `PROXY_AGENT_HEADER_VALUE` still needs the `v` :)

Author: abhinavsingh

.gitignore

@@ -26,4 +26,6 @@ cover
 htmlcov
 dist
 build
+
 proxy/public
+docs/_build

CODE_OF_CONDUCT.md

@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at [email protected]. All
+reported by contacting the project team at mailsforabhinav+proxy@gmail.com. All
 complaints will be reviewed and investigated and will result in a response that
 is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.

Makefile

@@ -2,9 +2,8 @@ SHELL := /bin/bash
 
 NS ?= abhinavsingh
 IMAGE_NAME ?= proxy.py
-VERSION ?= v$(shell ./scm-version.sh)
 LATEST_TAG := $(NS)/$(IMAGE_NAME):latest
-IMAGE_TAG := $(NS)/$(IMAGE_NAME):$(VERSION)
+IMAGE_TAG := $(NS)/$(IMAGE_NAME):$(shell ./write-scm-version.sh)
 
 HTTPS_KEY_FILE_PATH := https-key.pem
 HTTPS_CERT_FILE_PATH := https-cert.pem
@@ -17,12 +16,14 @@ CA_SIGNING_KEY_FILE_PATH := ca-signing-key.pem
 
 .PHONY: all https-certificates sign-https-certificates ca-certificates
 .PHONY: lib-check lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest
-.PHONY: lib-release-test lib-release lib-profile
-.PHONY: lib-dep, lib-flake8, lib-mypy, lib-scm-version
+.PHONY: lib-release-test lib-release lib-profile lib-doc
+.PHONY: lib-dep lib-flake8 lib-mypy
 .PHONY: container container-run container-release
 .PHONY: devtools dashboard dashboard-clean
 
-all: lib-test
+all:
+	echo $(IMAGE_TAG)
+	# lib-test
 
 https-certificates:
 	# Generate server key
@@ -82,14 +83,13 @@ lib-clean:
 	rm -rf .hypothesis
 
 lib-dep:
+	pip install --upgrade pip && \
 	pip install \
 		-r requirements.txt \
 		-r requirements-testing.txt \
 		-r requirements-release.txt \
-		-r requirements-tunnel.txt
-
-lib-scm-version:
-	@echo "version = '$(VERSION)'" > proxy/common/_scm_version.py
+		-r requirements-tunnel.txt \
+		-r docs/requirements.txt
 
 lib-lint:
 	python -m tox -e lint
@@ -114,6 +114,16 @@ lib-release-test: lib-package
 lib-release: lib-package
 	twine upload dist/*
 
+lib-doc:
+	pushd docs && \
+	python -m sphinx \
+		--keep-going \
+		-b dirhtml \
+		-d _build/doctrees \
+		-D language=en . _build/html && \
+	popd && \
+	open docs/_build/html/index.html
+
 lib-coverage:
 	pytest --cov=proxy --cov=tests --cov-report=html tests/
 	open htmlcov/index.html

README.md

@@ -379,7 +379,7 @@ To start `proxy.py` from source code follow these instructions:
 - Install deps
 
   ```console
-  ❯ make lib-dep lib-scm-version
+  ❯ make lib-dep
   ```
 
 - Optionally, run tests

proxy/http/methods.py

@@ -16,28 +16,91 @@
 from typing import NamedTuple
 
 
+# Ref: https://www.iana.org/assignments/http-methods/http-methods.xhtml
 HttpMethods = NamedTuple(
     'HttpMethods', [
+        ('ACL', bytes),
+        ('BASELINE_CONTROL', bytes),
+        ('BIND', bytes),
+        ('CHECKIN', bytes),
+        ('CHECKOUT', bytes),
+        ('CONNECT', bytes),
+        ('COPY', bytes),
+        ('DELETE', bytes),
         ('GET', bytes),
         ('HEAD', bytes),
+        ('LABEL', bytes),
+        ('LINK', bytes),
+        ('LOCK', bytes),
+        ('MERGE', bytes),
+        ('MKACTIVITY', bytes),
+        ('MKCALENDAR', bytes),
+        ('MKCOL', bytes),
+        ('MKREDIRECTREF', bytes),
+        ('MKWORKSPACE', bytes),
+        ('MOVE', bytes),
+        ('OPTIONS', bytes),
+        ('ORDERPATCH', bytes),
+        ('PATCH', bytes),
         ('POST', bytes),
+        ('PRI', bytes),
+        ('PROPFIND', bytes),
+        ('PROPPATCH', bytes),
         ('PUT', bytes),
-        ('DELETE', bytes),
-        ('CONNECT', bytes),
-        ('OPTIONS', bytes),
+        ('REBIND', bytes),
+        ('REPORT', bytes),
+        ('SEARCH', bytes),
         ('TRACE', bytes),
-        ('PATCH', bytes),
+        ('UNBIND', bytes),
+        ('UNCHECKOUT', bytes),
+        ('UNLINK', bytes),
+        ('UNLOCK', bytes),
+        ('UPDATE', bytes),
+        ('UPDATEREDIRECTREF', bytes),
+        ('VERSION_CONTROL', bytes),
+        ('STAR', bytes),
     ],
 )
 
 httpMethods = HttpMethods(
+    b'ACL',
+    b'BASELINE-CONTROL',
+    b'BIND',
+    b'CHECKIN',
+    b'CHECKOUT',
+    b'CONNECT',
+    b'COPY',
+    b'DELETE',
     b'GET',
     b'HEAD',
+    b'LABEL',
+    b'LINK',
+    b'LOCK',
+    b'MERGE',
+    b'MKACTIVITY',
+    b'MKCALENDAR',
+    b'MKCOL',
+    b'MKREDIRECTREF',
+    b'MKWORKSPACE',
+    b'MOVE',
+    b'OPTIONS',
+    b'ORDERPATCH',
+    b'PATCH',
     b'POST',
+    b'PRI',
+    b'PROPFIND',
+    b'PROPPATCH',
     b'PUT',
-    b'DELETE',
-    b'CONNECT',
-    b'OPTIONS',
+    b'REBIND',
+    b'REPORT',
+    b'SEARCH',
     b'TRACE',
-    b'PATCH',
+    b'UNBIND',
+    b'UNCHECKOUT',
+    b'UNLINK',
+    b'UNLOCK',
+    b'UPDATE',
+    b'UPDATEREDIRECTREF',
+    b'VERSION-CONTROL',
+    b'*',
 )

proxy/http/parser/parser.py

@@ -304,27 +304,25 @@ def _process_line_and_headers(self, raw: bytes) -> Tuple[bool, bytes]:
 
     def _process_line(self, raw: bytes) -> None:
         if self.type == httpParserTypes.REQUEST_PARSER:
-            # Ref:
-            # https://datatracker.ietf.org/doc/html/rfc2616#section-5.1
-            # https://greenbytes.de/tech/webdav/rfc7230.html#request.line
-            # https://greenbytes.de/tech/webdav/rfc7231.html#methods
-            # http://www.iana.org/assignments/http-methods/http-methods.xhtml
             if self.protocol is not None and self.protocol.version is None:
                 # We expect to receive entire proxy protocol v1 line
                 # in one network read and don't expect partial packets
                 self.protocol.parse(raw)
             else:
+                # Ref: https://datatracker.ietf.org/doc/html/rfc2616#section-5.1
                 line = raw.split(WHITESPACE)
                 if len(line) == 3:
                     self.method = line[0].upper()
                     self.set_url(line[1])
                     self.version = line[2]
                     self.state = httpParserStates.LINE_RCVD
                 else:
-                    # raise exception
-                    # TODO, it would be better to use raise HttpProtocolException,
-                    # but we should solve circular import problem first
-                    raise NotImplementedError('Invalid request line')
+                    # To avoid a possible attack vector, we raise exception
+                    # if parser receives an invalid request line.
+                    #
+                    # TODO: Better to use raise HttpProtocolException,
+                    # but we should solve circular import problem first.
+                    raise ValueError('Invalid request line')
         else:
             line = raw.split(WHITESPACE)
             self.version = line[0]

scm-version.sh

@@ -7,7 +7,7 @@ description = ⚡⚡⚡Fast, Lightweight, Pluggable, TLS interception capable pr
 long_description = file: README.md
 long_description_content_type = text/markdown
 author = Abhinav Singh
-author_email = [email protected]
+author_email = mailsforabhinav+proxy@gmail.com
 license = 'BSD'
 license_files =
   LICENSE.md

setup.cfg

@@ -24,10 +24,10 @@ def setUp(self) -> None:
         self.parser = HttpParser(httpParserTypes.REQUEST_PARSER)
 
     def test_issue_127(self) -> None:
-        with self.assertRaises(NotImplementedError):
+        with self.assertRaises(ValueError):
             self.parser.parse(CRLF)
 
-        with self.assertRaises(NotImplementedError):
+        with self.assertRaises(ValueError):
             raw = b'qwqrqw!@!#@!#ad adfad\r\n'
             while True:
                 self.parser.parse(raw)

tests/http/test_http_parser.py

@@ -397,7 +397,8 @@ def assert_data_queued(
             b'Via: 1.1 proxy.py v%s' % bytes_(__version__),
             CRLF,
         ])
-        server.queue.assert_called_once_with(pkt)
+        server.queue.assert_called_once()
+        self.assertEqual(server.queue.call_args_list[0][0][0].tobytes(), pkt)
         server.buffer_size.return_value = len(pkt)
 
     def assert_data_queued_to_server(self, server: mock.Mock) -> None:

tests/http/test_protocol_handler.py

@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# write-scm-version.sh exists because `proxy.py``
+# auto-detects it's next version from git.  Hence,
+# for `proxy.py` to work `proxy/common/_scm_version.py`
+# file must be auto-generated with necessary information.
+#
+# For CI/CD, this file is generated via `tox` integration.
+# For local development (without editable install), you
+# must run this script to pre-populate `_scm_version.py`.
+#
+# This file is integrated by default within `Makefile`.
+# For every make target invocation, `_scm_version.py` file
+# will be re-written.
+
+# Guessed Version 2.3.2.dev146+gad54132.d20211114
+VERSION=$(python -m setuptools_scm --version | \
+    # 2.3.2.dev146+gad54132.d20211114
+    awk '{print $3}')
+
+# Store default IFS
+OLDIFS=$IFS
+
+IFS="+"
+set -- $VERSION
+SEMVER=$1
+DATE_AND_HASH=$2
+
+IFS="."
+set -- $SEMVER
+MAJOR=$1
+MINOR=$2
+PATCH=$3
+DISTANCE=$4
+
+# Reset IFS
+IFS=$OLDIFS
+
+echo "# coding: utf-8
+# file generated by setuptools_scm
+# don't change, don't track in version control
+version = '${VERSION}'
+version_tuple = (${MAJOR}, ${MINOR}, ${PATCH}, '${DISTANCE}', '${DATE_AND_HASH}')" > \
+    proxy/common/_scm_version.py
+
+echo $MAJOR.$MINOR.$PATCH.$DISTANCE