Aiohttp raises an certificate error with some sites that browser opens normally

Question

code example

import aiohttp
import asyncio


async def main(url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as response:
            print("Status:", response.status)
            print("Content-type:", response.headers['content-type'])
            html = await response.text()
            print("Body:", html[:15], "...")


url = "https://shikimori.one/"

loop = asyncio.get_event_loop()
loop.run_until_complete(main(url))

traceback

    Traceback (most recent call last):
  File "D:\projects\parser\test\test_aiohttp.py", line 20, in <module>
    loop.run_until_complete(main(url))
  File "C:\Users\user\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 642, in run_until_complete
    return future.result()
  File "D:\projects\parser\test\test_aiohttp.py", line 8, in main
    async with session.get(url) as response:
  File "D:\projects\parser\venv\lib\site-packages\aiohttp\client.py", line 1117, in __aenter__
    self._resp = await self._coro
  File "D:\projects\parser\venv\lib\site-packages\aiohttp\client.py", line 520, in _request
    conn = await self._connector.connect(
  File "D:\projects\parser\venv\lib\site-packages\aiohttp\connector.py", line 535, in connect
    proto = await self._create_connection(req, traces, timeout)
  File "D:\projects\parser\venv\lib\site-packages\aiohttp\connector.py", line 892, in _create_connection
    _, proto = await self._create_direct_connection(req, traces, timeout)
  File "D:\projects\parser\venv\lib\site-packages\aiohttp\connector.py", line 1051, in _create_direct_connection
    raise last_exc
  File "D:\projects\parser\venv\lib\site-packages\aiohttp\connector.py", line 1020, in _create_direct_connection
    transp, proto = await self._wrap_create_connection(
  File "D:\projects\parser\venv\lib\site-packages\aiohttp\connector.py", line 971, in _wrap_create_connection
    raise ClientConnectorCertificateError(req.connection_key, exc) from exc
aiohttp.client_exceptions.ClientConnectorCertificateError: Cannot connect to host shikimori.one:443 ssl:True [SSLCertVerificationError: (1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1129)')]

my config:

• windows 10
• python 3.9.7
• aiohttp 3.7.4.post0

Everything works fine if I get request with ssl=False parameter. But I don't think this is the right solution to the problem.

Answer 1

so you need to install certificates for your python version. on mac, you can run the below in a terminal window and it should solve your problems.

 /Applications/Python\ 3.10/Install\ Certificates.command

Answer 2

I think I solved the problem. You can install certifi and use its certificates.

import aiohttp
import asyncio
import ssl
import certifi


async def main(url):
    ssl_context = ssl.create_default_context(cafile=certifi.where())
    conn = aiohttp.TCPConnector(ssl=ssl_context)

    async with aiohttp.ClientSession(connector=conn) as session:
        async with session.get(url) as response:
            print("Status:", response.status)
            

url = "https://shikimori.one/"

loop = asyncio.get_event_loop()
loop.run_until_complete(main(url))

if it doesn't help, you can try adding certificates manually as described here