Error not getting captured in except block

Question

I am trying to capture the error in Python for the below error but I am unable to do so:

from app_store_scraper import AppStore
import requests
import logging

logger = logging.getLogger()

try:
    minecraft = AppStore(country='sk', app_name='telekom')
    minecraft.review(how_many=20)
    reviews = (minecraft.reviews)
    print(reviews)
except AttributeError:
    logging.error("Please check the appname")
except requests.exceptions.ConnectionError:
    logging.error("Connection error1")
except requests.TooManyRedirects:
    logging.error("Connection error2")
except requests.RequestException:
    logging.error("Connection error3")
except Exception as e:
   logging.error("xxxxxxxxxxxxxxxxxxxxxxx")

Error:

 Something went wrong: HTTPSConnectionPool(host='amp-api.apps.apple.com', port=443): Max retries exceeded with url: /v1/catalog/sk/apps/1106471260/reviews?l=en-GB&offset=0&limit=20&platform=web&additionalPlatforms=appletv%2Cipad%2Ciphone%2Cmac (Caused by ResponseError('too many 404 error responses'))

But it's not getting captured in any of the except blocks.

Note: I am using apple app scraper library in Python to scrape data.

Answer 1

The basic idea is to capture the logger output emitted from within the app_store_scraper library (the logger defined by their Base class) and raise a manual exception for retry error:

from app_store_scraper import AppStore
import logging
from io import StringIO
import urllib3
import sys


tmp_stderr = StringIO()

log_stream = StringIO()
log_handler = logging.StreamHandler(stream=log_stream)
log_handler.setLevel(logging.ERROR)
loggerBase = logging.getLogger("Base")
base_handlers = [oh for oh in  loggerBase.handlers]

class ModifyLogger():
    def __init__(self, stder, tmpstrm):
        self.stder = stder
        self.tmpstrm = tmpstrm

    def __enter__(self):
        #logging.disable(logging.ERROR)
        sys.stderr = self.tmpstrm   # redirecting stderr
        
        # Removing existing handlers for "Base" logger and adding our own
        for oh in base_handlers:
            loggerBase.removeHandler(oh)
        loggerBase.addHandler(log_handler)
        loggerBase.setLevel(logging.ERROR)

    def __exit__(self, exit_type, exit_value, exit_traceback):
        #logging.disable(logging.NOTSET)
        # Removing our handler for "Base" logger and adding originals back
        loggerBase.removeHandler(log_handler)
        for oh in base_handlers:
            loggerBase.addHandler(oh)
        
        sys.stderr = self.stder   # restoring stderr


with ModifyLogger(sys.stderr, tmp_stderr):      # a context manager
    try:
        minecraft = AppStore(country='sk', app_name='telekom')
        minecraft.review(how_many=20)
        reviews = (minecraft.reviews)
        print(reviews)

        logs = log_stream.getvalue()
        if len(logs.split('\n'))==2 and logs.find('HTTPSConnectionPool', 0, 100) and logs.find('Max retries exceeded', 0, -1):
            # Extracting the URL
            url1 = logs.split('=')[1].split(',')[0].strip("'").strip()
            url2 = logs.split(':')[3].split('(')[0].strip()
            # Raising a manual exception for maximum retries
            raise urllib3.exceptions.MaxRetryError(pool='HTTPSConnectionPool', url=url1+url2, reason='Max Retries exceeded')

    except urllib3.exceptions.MaxRetryError as e:
        print('\"{}\" while accessing URL: {}'.format(e.reason, e.url))

The output from above code:

[]
"Max Retries exceeded" while accessing URL: amp-api.apps.apple.com/v1/catalog/sk/apps/1106471260/reviews?l=en-GB&offset=0&limit=20&platform=web&additionalPlatforms=appletv%2Cipad%2Ciphone%2Cmac