pyscalpel.http

This module contains objects representing HTTP objects passed to the user's hooks

View Source

 1"""
 2    This module contains objects representing HTTP objects passed to the user's hooks
 3"""
 4
 5from .request import Request, Headers
 6from .response import Response
 7from .flow import Flow
 8from .utils import match_patterns, host_is
 9from . import body
10
11__all__ = [
12    "body",  # <- pdoc shows a warning for this declaration but won't display it when absent
13    "Request",
14    "Response",
15    "Headers",
16    "Flow",
17    "host_is",
18    "match_patterns",
19]

class Response(_internal_mitmproxy.http.Response): View Source

 22class Response(MITMProxyResponse):
 23    """A "Burp oriented" HTTP response class
 24
 25
 26    This class allows to manipulate Burp responses in a Pythonic way.
 27
 28    Fields:
 29        scheme: http or https
 30        host: The initiating request target host
 31        port: The initiating request target port
 32        request: The initiating request.
 33    """
 34
 35    scheme: Literal["http", "https"] = "http"
 36    host: str = ""
 37    port: int = 0
 38    request: Request | None = None
 39
 40    def __init__(
 41        self,
 42        http_version: bytes,
 43        status_code: int,
 44        reason: bytes,
 45        headers: Headers | tuple[tuple[bytes, bytes], ...],
 46        content: bytes | None,
 47        trailers: Headers | tuple[tuple[bytes, bytes], ...] | None,
 48        scheme: Literal["http", "https"] = "http",
 49        host: str = "",
 50        port: int = 0,
 51    ):
 52        # Construct the base/inherited MITMProxy response.
 53        super().__init__(
 54            http_version,
 55            status_code,
 56            reason,
 57            headers,
 58            content,
 59            trailers,
 60            timestamp_start=time.time(),
 61            timestamp_end=time.time(),
 62        )
 63        self.scheme = scheme
 64        self.host = host
 65        self.port = port
 66
 67    @classmethod
 68    # https://docs.mitmproxy.org/stable/api/mitmproxy/http.html#Response
 69    # link to mitmproxy documentation
 70    def from_mitmproxy(cls, response: MITMProxyResponse) -> Response:
 71        """Construct an instance of the Response class from a [mitmproxy.http.HTTPResponse](https://docs.mitmproxy.org/stable/api/mitmproxy/http.html#Response).
 72        :param response: The [mitmproxy.http.HTTPResponse](https://docs.mitmproxy.org/stable/api/mitmproxy/http.html#Response) to convert.
 73        :return: A :class:`Response` with the same data as the [mitmproxy.http.HTTPResponse](https://docs.mitmproxy.org/stable/api/mitmproxy/http.html#Response).
 74        """
 75        return cls(
 76            always_bytes(response.http_version),
 77            response.status_code,
 78            always_bytes(response.reason),
 79            Headers.from_mitmproxy(response.headers),
 80            response.content,
 81            Headers.from_mitmproxy(response.trailers) if response.trailers else None,
 82        )
 83
 84    @classmethod
 85    def from_burp(
 86        cls,
 87        response: IHttpResponse,
 88        service: IHttpService | None = None,
 89        request: IHttpRequest | None = None,
 90    ) -> Response:
 91        """Construct an instance of the Response class from a Burp suite :class:`IHttpResponse`."""
 92        body = get_bytes(cast(IByteArray, response.body())) if response.body() else b""
 93        scalpel_response = cls(
 94            always_bytes(response.httpVersion() or "HTTP/1.1"),
 95            response.statusCode(),
 96            always_bytes(response.reasonPhrase() or b""),
 97            Headers.from_burp(response.headers()),
 98            body,
 99            None,
100        )
101
102        burp_request: IHttpRequest | None = request
103        if burp_request is None:
104            try:
105                # Some responses can have a "initiatingRequest" field.
106                # https://portswigger.github.io/burp-extensions-montoya-api/javadoc/burp/api/montoya/http/handler/HttpResponseReceived.html#initiatingRequest():~:text=HttpRequest-,initiatingRequest(),-Returns%3A
107                burp_request = response.initiatingRequest()  # type: ignore
108            except AttributeError:
109                pass
110
111        if burp_request:
112            scalpel_response.request = Request.from_burp(burp_request, service)
113
114        if not service and burp_request:
115            # The only way to check if the Java method exist without writing Java is catching the error.
116            service = burp_request.httpService()
117
118        if service:
119            scalpel_response.scheme = "https" if service.secure() else "http"
120            scalpel_response.host = service.host()
121            scalpel_response.port = service.port()
122
123        return scalpel_response
124
125    def __bytes__(self) -> bytes:
126        """Convert the response to raw bytes."""
127        # Reserialize the response to bytes.
128
129        # Format the first line of the response. (e.g. "HTTP/1.1 200 OK\r\n")
130        first_line = (
131            b" ".join(
132                always_bytes(s)
133                for s in (self.http_version, str(self.status_code), self.reason)
134            )
135            + b"\r\n"
136        )
137
138        # Format the response's headers part.
139        headers_lines = b"".join(
140            b"%s: %s\r\n" % (key, val) for key, val in self.headers.fields
141        )
142
143        # Set a default value for the response's body. (None -> b"")
144        body = self.content or b""
145
146        # Build the whole response and return it.
147        return first_line + headers_lines + b"\r\n" + body
148
149    def to_burp(self) -> IHttpResponse:  # pragma: no cover (uses Java API)
150        """Convert the response to a Burp suite :class:`IHttpResponse`."""
151        response_byte_array: IByteArray = PythonUtils.toByteArray(bytes(self))
152
153        return HttpResponse.httpResponse(response_byte_array)
154
155    @classmethod
156    def from_raw(
157        cls, data: bytes | str
158    ) -> Response:  # pragma: no cover (uses Java API)
159        """Construct an instance of the Response class from raw bytes.
160        :param data: The raw bytes to convert.
161        :return: A :class:`Response` parsed from the raw bytes.
162        """
163        # Use the Burp API to trivialize the parsing of the response from raw bytes.
164        # Convert the raw bytes to a Burp ByteArray.
165        # Plain strings are OK too.
166        str_or_byte_array: IByteArray | str = (
167            data if isinstance(data, str) else PythonUtils.toByteArray(data)
168        )
169
170        # Instantiate a new Burp HTTP response.
171        burp_response: IHttpResponse = HttpResponse.httpResponse(str_or_byte_array)
172
173        return cls.from_burp(burp_response)
174
175    @classmethod
176    def make(
177        cls,
178        status_code: int = 200,
179        content: bytes | str = b"",
180        headers: Headers | tuple[tuple[bytes, bytes], ...] = (),
181        host: str = "",
182        port: int = 0,
183        scheme: Literal["http", "https"] = "http",
184    ) -> "Response":
185        # Use the base/inherited make method to construct a MITMProxy response.
186        mitmproxy_res = MITMProxyResponse.make(status_code, content, headers)
187
188        res = cls.from_mitmproxy(mitmproxy_res)
189        res.host = host
190        res.scheme = scheme
191        res.port = port
192
193        return res
194
195    def host_is(self, *patterns: str) -> bool:
196        """Matches the host against the provided patterns
197
198        Returns:
199            bool: Whether at least one pattern matched
200        """
201        return host_is(self.host, *patterns)
202
203    @property
204    def body(self) -> bytes | None:
205        """Alias for content()
206
207        Returns:
208            bytes | None: The request body / content
209        """
210        return self.content
211
212    @body.setter
213    def body(self, val: bytes | None):
214        self.content = val

A "Burp oriented" HTTP response class

This class allows to manipulate Burp responses in a Pythonic way.

Fields: scheme: http or https host: The initiating request target host port: The initiating request target port request: The initiating request.

Response( http_version: bytes, status_code: int, reason: bytes, headers: Headers | tuple[tuple[bytes, bytes], ...], content: bytes | None, trailers: Headers | tuple[tuple[bytes, bytes], ...] | None, scheme: Literal['http', 'https'] = 'http', host: str = '', port: int = 0) View Source

40    def __init__(
41        self,
42        http_version: bytes,
43        status_code: int,
44        reason: bytes,
45        headers: Headers | tuple[tuple[bytes, bytes], ...],
46        content: bytes | None,
47        trailers: Headers | tuple[tuple[bytes, bytes], ...] | None,
48        scheme: Literal["http", "https"] = "http",
49        host: str = "",
50        port: int = 0,
51    ):
52        # Construct the base/inherited MITMProxy response.
53        super().__init__(
54            http_version,
55            status_code,
56            reason,
57            headers,
58            content,
59            trailers,
60            timestamp_start=time.time(),
61            timestamp_end=time.time(),
62        )
63        self.scheme = scheme
64        self.host = host
65        self.port = port

scheme: Literal['http', 'https'] = 'http'

host: str = ''

port: int = 0

request: Request | None = None

@classmethod

def from_mitmproxy( cls, response: _internal_mitmproxy.http.Response) -> Response: View Source

67    @classmethod
68    # https://docs.mitmproxy.org/stable/api/mitmproxy/http.html#Response
69    # link to mitmproxy documentation
70    def from_mitmproxy(cls, response: MITMProxyResponse) -> Response:
71        """Construct an instance of the Response class from a [mitmproxy.http.HTTPResponse](https://docs.mitmproxy.org/stable/api/mitmproxy/http.html#Response).
72        :param response: The [mitmproxy.http.HTTPResponse](https://docs.mitmproxy.org/stable/api/mitmproxy/http.html#Response) to convert.
73        :return: A :class:`Response` with the same data as the [mitmproxy.http.HTTPResponse](https://docs.mitmproxy.org/stable/api/mitmproxy/http.html#Response).
74        """
75        return cls(
76            always_bytes(response.http_version),
77            response.status_code,
78            always_bytes(response.reason),
79            Headers.from_mitmproxy(response.headers),
80            response.content,
81            Headers.from_mitmproxy(response.trailers) if response.trailers else None,
82        )

Construct an instance of the Response class from a mitmproxy.http.HTTPResponse.

# Parameters

**response: The mitmproxy.http.HTTPResponse to convert.

# Returns

A Response with the same data as the mitmproxy.http.HTTPResponse.

@classmethod

def from_burp( cls, response: pyscalpel.java.burp.http_response.IHttpResponse, service: pyscalpel.java.burp.http_service.IHttpService | None = None, request: pyscalpel.java.burp.http_request.IHttpRequest | None = None) -> Response: View Source

 84    @classmethod
 85    def from_burp(
 86        cls,
 87        response: IHttpResponse,
 88        service: IHttpService | None = None,
 89        request: IHttpRequest | None = None,
 90    ) -> Response:
 91        """Construct an instance of the Response class from a Burp suite :class:`IHttpResponse`."""
 92        body = get_bytes(cast(IByteArray, response.body())) if response.body() else b""
 93        scalpel_response = cls(
 94            always_bytes(response.httpVersion() or "HTTP/1.1"),
 95            response.statusCode(),
 96            always_bytes(response.reasonPhrase() or b""),
 97            Headers.from_burp(response.headers()),
 98            body,
 99            None,
100        )
101
102        burp_request: IHttpRequest | None = request
103        if burp_request is None:
104            try:
105                # Some responses can have a "initiatingRequest" field.
106                # https://portswigger.github.io/burp-extensions-montoya-api/javadoc/burp/api/montoya/http/handler/HttpResponseReceived.html#initiatingRequest():~:text=HttpRequest-,initiatingRequest(),-Returns%3A
107                burp_request = response.initiatingRequest()  # type: ignore
108            except AttributeError:
109                pass
110
111        if burp_request:
112            scalpel_response.request = Request.from_burp(burp_request, service)
113
114        if not service and burp_request:
115            # The only way to check if the Java method exist without writing Java is catching the error.
116            service = burp_request.httpService()
117
118        if service:
119            scalpel_response.scheme = "https" if service.secure() else "http"
120            scalpel_response.host = service.host()
121            scalpel_response.port = service.port()
122
123        return scalpel_response

Construct an instance of the Response class from a Burp suite IHttpResponse.

def to_burp(self) -> pyscalpel.java.burp.http_response.IHttpResponse: View Source

149    def to_burp(self) -> IHttpResponse:  # pragma: no cover (uses Java API)
150        """Convert the response to a Burp suite :class:`IHttpResponse`."""
151        response_byte_array: IByteArray = PythonUtils.toByteArray(bytes(self))
152
153        return HttpResponse.httpResponse(response_byte_array)

Convert the response to a Burp suite IHttpResponse.

@classmethod

def from_raw(cls, data: bytes | str) -> Response: View Source

155    @classmethod
156    def from_raw(
157        cls, data: bytes | str
158    ) -> Response:  # pragma: no cover (uses Java API)
159        """Construct an instance of the Response class from raw bytes.
160        :param data: The raw bytes to convert.
161        :return: A :class:`Response` parsed from the raw bytes.
162        """
163        # Use the Burp API to trivialize the parsing of the response from raw bytes.
164        # Convert the raw bytes to a Burp ByteArray.
165        # Plain strings are OK too.
166        str_or_byte_array: IByteArray | str = (
167            data if isinstance(data, str) else PythonUtils.toByteArray(data)
168        )
169
170        # Instantiate a new Burp HTTP response.
171        burp_response: IHttpResponse = HttpResponse.httpResponse(str_or_byte_array)
172
173        return cls.from_burp(burp_response)

Construct an instance of the Response class from raw bytes.

# Parameters

data: The raw bytes to convert.

# Returns

A Response parsed from the raw bytes.

@classmethod

def make( cls, status_code: int = 200, content: bytes | str = b'', headers: Headers | tuple[tuple[bytes, bytes], ...] = (), host: str = '', port: int = 0, scheme: Literal['http', 'https'] = 'http') -> Response: View Source

175    @classmethod
176    def make(
177        cls,
178        status_code: int = 200,
179        content: bytes | str = b"",
180        headers: Headers | tuple[tuple[bytes, bytes], ...] = (),
181        host: str = "",
182        port: int = 0,
183        scheme: Literal["http", "https"] = "http",
184    ) -> "Response":
185        # Use the base/inherited make method to construct a MITMProxy response.
186        mitmproxy_res = MITMProxyResponse.make(status_code, content, headers)
187
188        res = cls.from_mitmproxy(mitmproxy_res)
189        res.host = host
190        res.scheme = scheme
191        res.port = port
192
193        return res

Simplified API for creating response objects.

def host_is(self, *patterns: str) -> bool: View Source

195    def host_is(self, *patterns: str) -> bool:
196        """Matches the host against the provided patterns
197
198        Returns:
199            bool: Whether at least one pattern matched
200        """
201        return host_is(self.host, *patterns)

Matches the host against the provided patterns

Returns: bool: Whether at least one pattern matched

body: bytes | None View Source

203    @property
204    def body(self) -> bytes | None:
205        """Alias for content()
206
207        Returns:
208            bytes | None: The request body / content
209        """
210        return self.content

Alias for content()

Returns: bytes | None: The request body / content

Inherited Members

_internal_mitmproxy.http.Response: data; status_code; reason; cookies; refresh
_internal_mitmproxy.http.Message: from_state; get_state; set_state; stream; http_version; is_http10; is_http11; is_http2; headers; trailers; raw_content; content; text; set_content; get_content; set_text; get_text; timestamp_start; timestamp_end; decode; encode; json
_internal_mitmproxy.coretypes.serializable.Serializable: copy

class Headers(_internal_mitmproxy.coretypes.multidict._MultiDict[~KT, ~VT], _internal_mitmproxy.coretypes.serializable.Serializable): View Source

16class Headers(MITMProxyHeaders):
17    """A wrapper around the MITMProxy Headers.
18
19    This class provides additional methods for converting headers between Burp suite and MITMProxy formats.
20    """
21
22    def __init__(self, fields: Iterable[tuple[bytes, bytes]] | None = None, **headers):
23        """
24        :param fields: The headers to construct the from.
25        :param headers: The headers to construct the from.
26        """
27        # Cannot safely use [] as default param because the reference will be shared with all __init__ calls
28        fields = fields or []
29
30        # Construct the base/inherited MITMProxy headers.
31        super().__init__(fields, **headers)
32
33    @classmethod
34    def from_mitmproxy(cls, headers: MITMProxyHeaders) -> Headers:
35        """
36        Creates a `Headers` from a `mitmproxy.http.Headers`.
37
38        :param headers: The `mitmproxy.http.Headers` to convert.
39        :type headers: :class Headers <https://docs.mitmproxy.org/stable/api/mitmproxy/http.html#Headers>`
40        :return: A `Headers` with the same headers as the `mitmproxy.http.Headers`.
41        """
42
43        # Construct from the raw MITMProxy headers data.
44        return cls(headers.fields)
45
46    @classmethod
47    def from_burp(cls, headers: list[IHttpHeader]) -> Headers:  # pragma: no cover
48        """Construct an instance of the Headers class from a Burp suite HttpHeader array.
49        :param headers: The Burp suite HttpHeader array to convert.
50        :return: A Headers with the same headers as the Burp suite HttpHeader array.
51        """
52
53        # print(f"burp: {headers}")
54        # Convert the list of Burp IHttpHeaders to a list of tuples: (key, value)
55        return cls(
56            (
57                (
58                    always_bytes(header.name()),
59                    always_bytes(header.value()),
60                )
61                for header in headers
62            )
63        )
64
65    def to_burp(self) -> list[IHttpHeader]:  # pragma: no cover
66        """Convert the headers to a Burp suite HttpHeader array.
67        :return: A Burp suite HttpHeader array.
68        """
69
70        # Convert the list of tuples: (key, value) to a list of Burp IHttpHeaders
71        return [
72            HttpHeader.httpHeader(always_str(header[0]), always_str(header[1]))
73            for header in self.fields
74        ]

A wrapper around the MITMProxy Headers.

This class provides additional methods for converting headers between Burp suite and MITMProxy formats.

Headers(fields: Optional[Iterable[tuple[bytes, bytes]]] = None, **headers) View Source

22    def __init__(self, fields: Iterable[tuple[bytes, bytes]] | None = None, **headers):
23        """
24        :param fields: The headers to construct the from.
25        :param headers: The headers to construct the from.
26        """
27        # Cannot safely use [] as default param because the reference will be shared with all __init__ calls
28        fields = fields or []
29
30        # Construct the base/inherited MITMProxy headers.
31        super().__init__(fields, **headers)

# Parameters

fields: The headers to construct the from.
headers: The headers to construct the from.

@classmethod

def from_mitmproxy( cls, headers: _internal_mitmproxy.http.Headers) -> Headers: View Source

33    @classmethod
34    def from_mitmproxy(cls, headers: MITMProxyHeaders) -> Headers:
35        """
36        Creates a `Headers` from a `mitmproxy.http.Headers`.
37
38        :param headers: The `mitmproxy.http.Headers` to convert.
39        :type headers: :class Headers <https://docs.mitmproxy.org/stable/api/mitmproxy/http.html#Headers>`
40        :return: A `Headers` with the same headers as the `mitmproxy.http.Headers`.
41        """
42
43        # Construct from the raw MITMProxy headers data.
44        return cls(headers.fields)

Creates a Headers from a mitmproxy.http.Headers.

# Parameters

headers: The mitmproxy.http.Headers to convert.

# Returns

A Headers with the same headers as the mitmproxy.http.Headers.

@classmethod

def from_burp( cls, headers: list[pyscalpel.java.burp.http_header.IHttpHeader]) -> Headers: View Source

46    @classmethod
47    def from_burp(cls, headers: list[IHttpHeader]) -> Headers:  # pragma: no cover
48        """Construct an instance of the Headers class from a Burp suite HttpHeader array.
49        :param headers: The Burp suite HttpHeader array to convert.
50        :return: A Headers with the same headers as the Burp suite HttpHeader array.
51        """
52
53        # print(f"burp: {headers}")
54        # Convert the list of Burp IHttpHeaders to a list of tuples: (key, value)
55        return cls(
56            (
57                (
58                    always_bytes(header.name()),
59                    always_bytes(header.value()),
60                )
61                for header in headers
62            )
63        )

Construct an instance of the Headers class from a Burp suite HttpHeader array.

# Parameters

headers: The Burp suite HttpHeader array to convert.

# Returns

A Headers with the same headers as the Burp suite HttpHeader array.

def to_burp(self) -> list[pyscalpel.java.burp.http_header.IHttpHeader]: View Source

65    def to_burp(self) -> list[IHttpHeader]:  # pragma: no cover
66        """Convert the headers to a Burp suite HttpHeader array.
67        :return: A Burp suite HttpHeader array.
68        """
69
70        # Convert the list of tuples: (key, value) to a list of Burp IHttpHeaders
71        return [
72            HttpHeader.httpHeader(always_str(header[0]), always_str(header[1]))
73            for header in self.fields
74        ]

Convert the headers to a Burp suite HttpHeader array.

# Returns

A Burp suite HttpHeader array.

Inherited Members

_internal_mitmproxy.coretypes.multidict._MultiDict: fields; get_all; set_all; add; insert; keys; values; items
_internal_mitmproxy.coretypes.serializable.Serializable: from_state; get_state; set_state; copy
collections.abc.MutableMapping: pop; popitem; clear; update; setdefault
collections.abc.Mapping: get

class Flow: View Source

10class Flow:
11    """Contains request and response and some utilities for match()"""
12
13    def __init__(
14        self,
15        scheme: Literal["http", "https"] = "http",
16        host: str = "",
17        port: int = 0,
18        request: Request | None = None,
19        response: Response | None = None,
20        text: bytes | None = None,
21    ):
22        self.scheme = scheme
23        self.host = host
24        self.port = port
25        self.request = request
26        self.response = response
27        self.text = text
28
29    def host_is(self, *patterns: str) -> bool:
30        """Matches a wildcard pattern against the target host
31
32        Returns:
33            bool: True if at least one pattern matched
34        """
35        return host_is(self.host, *patterns)
36
37    def path_is(self, *patterns: str) -> bool:
38        """Matches a wildcard pattern against the request path
39
40        Includes query string `?` and fragment `#`
41
42        Returns:
43            bool: True if at least one pattern matched
44        """
45        req = self.request
46        if req is None:
47            return False
48
49        return req.path_is(*patterns)

Contains request and response and some utilities for match()

Flow( scheme: Literal['http', 'https'] = 'http', host: str = '', port: int = 0, request: Request | None = None, response: Response | None = None, text: bytes | None = None) View Source

13    def __init__(
14        self,
15        scheme: Literal["http", "https"] = "http",
16        host: str = "",
17        port: int = 0,
18        request: Request | None = None,
19        response: Response | None = None,
20        text: bytes | None = None,
21    ):
22        self.scheme = scheme
23        self.host = host
24        self.port = port
25        self.request = request
26        self.response = response
27        self.text = text

scheme

host

port

request

response

text

def host_is(self, *patterns: str) -> bool: View Source

29    def host_is(self, *patterns: str) -> bool:
30        """Matches a wildcard pattern against the target host
31
32        Returns:
33            bool: True if at least one pattern matched
34        """
35        return host_is(self.host, *patterns)

Matches a wildcard pattern against the target host

Returns: bool: True if at least one pattern matched

def path_is(self, *patterns: str) -> bool: View Source

37    def path_is(self, *patterns: str) -> bool:
38        """Matches a wildcard pattern against the request path
39
40        Includes query string `?` and fragment `#`
41
42        Returns:
43            bool: True if at least one pattern matched
44        """
45        req = self.request
46        if req is None:
47            return False
48
49        return req.path_is(*patterns)

Matches a wildcard pattern against the request path

Includes query string ? and fragment #

Returns: bool: True if at least one pattern matched

def host_is(host: str, *patterns: str) -> bool: View Source

21def host_is(host: str, *patterns: str) -> bool:
22    """Matches a host using unix-like wildcard matching against multiple patterns
23
24    Args:
25        host (str): The host to match against
26        patterns (str): The patterns to use
27
28    Returns:
29        bool: The match result (True if at least one pattern matches, else False)
30    """
31    return match_patterns(host, *patterns)

Matches a host using unix-like wildcard matching against multiple patterns

Args: host (str): The host to match against patterns (str): The patterns to use

Returns: bool: The match result (True if at least one pattern matches, else False)

def match_patterns(to_match: str, *patterns: str) -> bool: View Source

 5def match_patterns(to_match: str, *patterns: str) -> bool:
 6    """Matches a string using unix-like wildcard matching against multiple patterns
 7
 8    Args:
 9        to_match (str): The string to match against
10        patterns (str): The patterns to use
11
12    Returns:
13        bool: The match result (True if at least one pattern matches, else False)
14    """
15    for pattern in patterns:
16        if fnmatch(to_match, pattern):
17            return True
18    return False

Matches a string using unix-like wildcard matching against multiple patterns

Args: to_match (str): The string to match against patterns (str): The patterns to use

Returns: bool: The match result (True if at least one pattern matches, else False)