2Make HTTP requests using the standard library.
5from pathlib import Path
13__all__ = ["build_request", "download_image", "fetch_url"]
16ssl_context = ssl.create_default_context(cafile=certifi.where())
18QueryParams = dict[str, str] | list[tuple[str, str]]
19Headers = dict[str, str]
23 url: str, *, params: QueryParams | None = None, headers: Headers | None = None
24) -> urllib.request.Request:
26 Build a urllib Request, appending query parameters and attaching headers.
28 if params is not None:
29 params_list = list(params.items()) if isinstance(params, dict) else params
31 u = urllib.parse.urlsplit(url)
32 query = urllib.parse.parse_qsl(u.query) + params_list
33 new_query = urllib.parse.urlencode(query)
34 url = urllib.parse.urlunsplit(
35 (u.scheme, u.netloc, u.path, new_query, u.fragment)
38 req = urllib.request.Request(url, headers=headers or {})
44 url: str, *, params: QueryParams | None = None, headers: Headers | None = None
47 Fetch the contents of the given URL and return the body of
50 req = build_request(url, params=params, headers=headers)
52 with urllib.request.urlopen(req, context=ssl_context) as resp:
53 data: bytes = resp.read()
58def choose_filename_extension(image_data: bytes, content_type: str | None) -> str:
60 Choose a filename extension for an image downloaded with the given
63 # First inspect the file signature to see if we can detect it
64 # as being a particular image format.
66 # This isn't an exhaustive set of file signatures; just examples I've
67 # personally encountered.
68 if image_data.startswith(b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01"):
71 # If that fails, look at the value of the Content-Type header.
72 if content_type is None:
73 raise ValueError("no Content-Type header, cannot determine image format")
75 content_type_mapping = {
83 return content_type_mapping[content_type]
85 raise ValueError(f"unrecognised Content-Type header: {content_type}")
92 params: QueryParams | None = None,
93 headers: Headers | None = None,
96 Download an image from the given URL to the target path, and return
97 the path of the downloaded file.
99 Add the appropriate file extension, based on the image's Content-Type.
101 Throws a FileExistsError if you try to overwrite an existing file.
103 req = build_request(url, params=params, headers=headers)
105 with urllib.request.urlopen(req, context=ssl_context) as resp:
106 image_data: bytes = resp.read()
108 image_format = choose_filename_extension(
109 image_data, content_type=resp.headers["content-type"]
112 out_path = out_prefix.with_suffix("." + image_format)
113 out_path.parent.mkdir(exist_ok=True, parents=True)
114 with open(out_path, "xb") as out_file:
115 out_file.write(image_data)