Executing HTTP requests using Asyncio

Question

The following function needs to send request asynchronously:

No. of rows in devices.csv = 1 million

Required: send a POST request for all 1 million rows per day for 3 days

def check_in():
    logging.info('Starting')
    day = 0
    while day < 3:
        logging.info('Check-in Day = ' + str(day))
        with open('devices.csv', newline='') as csvfile: 
            reader = csv.DictReader(csvfile)
            for row in reader:
                device_sn = row['serial_number']
                payload = {
                    "product": "##",
                    "board": "###",
                    "hardware_id": "0000",
                    "usage_id": "000",
                    "mac_address": row['mac_address'],
                    "serial_number": row['serial_number']
                }
                logging.info(
                    'Check-in device: ' + device_sn)
                checkin_post(payload, device_sn)
            day += 1

def checkin_post(payload, device_sn):
    payload = payload
    serial_number = device_sn

    print('\n' + 72 * '=' + '\nPOST /device/' +
          serial_number + '/check-in')

    resp = requests.post(base_url + '/device/' +
                         serial_number + '/check-in', auth=auth, json=payload)

    print(resp.status_code)

The code might change to probably something like:

async def checkin_post(payload, device_sn):

    payload = payload
    serial_number = device_sn

    print('\n' + 72 * '=' + '\nPOST /device/' +
          serial_number + '/check-in')

    resp = requests.post(base_url + '/device/' +
                         serial_number + '/check-in', auth=auth, json=payload)
    return resp.status_code


async def main(payload, device_sn):
    checkin_post(payload, device_sn)

Also, since there is no await, it isn't truly asynchronous.

as it stands this question is too broad. have you tried converting it from requests to asyncio yourself? — wpercy
– wpercy, Commented Oct 14, 2019 at 21:31
@wpercy I have posted what I tried, I am sure Its incorrect. — Harsh
– Harsh, Commented Oct 14, 2019 at 21:41
either you execute the call to requests within a thread pool or you look at aiohttp — gelonida
– gelonida, Commented Oct 14, 2019 at 22:38

Pratibha Gupta · Accepted Answer · 2019-10-21 00:01:11Z

Sorry for replying late if you already got it fixed by now. But you need to return ensure_future object. Can you try following code:

async def check_in():
    logging.info('Starting')
    count = 0
    futures = []
    while count < 3:
        logging.info('Check-in Day = ' + str(count))
        with open('devices.csv', newline='') as csvfile:
            reader = csv.DictReader(csvfile)
            for row in reader:
                device_sn = row['serial_number']
                payload = {
                    "product": "##",
                    "board": "###",
                    "hardware_id": "0000",
                    "usage_id": "000",
                    "mac_address": row['mac_address'],
                    "serial_number": row['serial_number']
                }
                logging.info(
                    'Check-in device: ' + device_sn)
                async with aiohttp.ClientSession() as session:
                    async with session.post(base_url + '/device/' + device_sn + '/check-in', auth=auth, json=payload) as resp:
                        futures.append(asyncio.ensure_future(await resp.status))
            count += 1
    return await asyncio.gather(*futures)

def main():
    loop = asyncio.get_event_loop()
    futures = asyncio.ensure_future(check_in())
    responses = loop.run_until_complete(futures)

I have updated my question, payload and serial_number are passed from another function.
have you tried removing print(resp.status) statement? probably you can print status before waiting for the response.
Please look at my trials in the answers below. this doesn't work too.
Please edit your original question instead of posting new answers: stackoverflow.com/help/editing
@jtbandes thanks for reviewing it. Edited my original answer.

Collectives™ on Stack Overflow

Executing HTTP requests using Asyncio

1 Answer 1

5 Comments

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

5 Comments

Related