Return to Question

I've been working at speeding up my web scraping with the asyncio library. I have a working solution, but am unsure as to how pythonic it is or if I am properly using the library. Any input would be appreciated.

import aiohttp
import asyncio
import requests
from lxml import etree


@asyncio.coroutine
def get(*args, **kwargs):
    """
    A wrapper method for aiohttp's get method. Taken from Georges Dubus' article at
    http://compiletoi.net/fast-scraping-in-python-with-asyncio.html
    """
    response = yield from aiohttp.request('GET', *args, **kwargs)
    return (yield from response.read_and_close())


@asyncio.coroutine
def extract_text(url):
    """
    Given the url for a chapter, extract the relevant text from it
    :param url: the url for the chapter to scrape
    :return: a string containing the chapter's text
    """
    sem = asyncio.Semaphore(5)
    with (yield from sem):
        page = yield from get(url)

    tree = etree.HTML(page)
    paragraphs = tree.findall('.//*/div[@class="entry-content"]/p')[1: -1]
    return b'\n'.join(etree.tostring(paragraph) for paragraph in paragraphs)


def generate_links():
    """
    Generate the links to each of the chapters
    :return: A list of strings containing every url to visit
    """
    start_url = 'https://twigserial.wordpress.com/'
    base_url = 'https://twigserial.wordpress.com/category/story/'
    tree = etree.HTML(requests.get(start_url).text)
    xpath = './/*/option[@class="level-2"]/text()'
    return [base_url + suffix.strip() for suffix in tree.xpath(xpath)]


@asyncio.coroutine
def run():
    links = generate_links()
    chapters = []

    for f in asyncio.as_completed([extract_text(link) for link in links]):
        result = yield from f
        chapters.append(result)

    return chapters


def main():
    loop = asyncio.get_event_loop()
    chapters = loop.run_until_complete(run())
    print(len(chapters))


if __name__ == '__main__':
    main()

I've been working at speeding up my web scraping with the asyncio library. I have a working solution, but am unsure as to how pythonic it is or if I am properly using the library. Any input would be appreciated.

import aiohttp
import asyncio
import requests
from lxml import etree


@asyncio.coroutine
def get(*args, **kwargs):
    """
    A wrapper method for aiohttp's get method. Taken from Georges Dubus' article at
    http://compiletoi.net/fast-scraping-in-python-with-asyncio.html
    """
    response = yield from aiohttp.request('GET', *args, **kwargs)
    return (yield from response.read_and_close())


@asyncio.coroutine
def extract_text(url):
    """
    Given the url for a chapter, extract the relevant text from it
    :param url: the url for the chapter to scrape
    :return: a string containing the chapter's text
    """
    sem = asyncio.Semaphore(5)
    with (yield from sem):
        page = yield from get(url)

    tree = etree.HTML(page)
    paragraphs = tree.findall('.//*/div[@class="entry-content"]/p')[1: -1]
    return b'\n'.join(etree.tostring(paragraph) for paragraph in paragraphs)


def generate_links():
    """
    Generate the links to each of the chapters
    :return: A list of strings containing every url to visit
    """
    start_url = 'https://twigserial.wordpress.com/'
    base_url = 'https://twigserial.wordpress.com/category/story/'
    tree = etree.HTML(requests.get(start_url).text)
    xpath = './/*/option[@class="level-2"]/text()'
    return [base_url + suffix.strip() for suffix in tree.xpath(xpath)]


@asyncio.coroutine
def run():
    links = generate_links()
    chapters = []

    for f in asyncio.as_completed([extract_text(link) for link in links]):
        result = yield from f
        chapters.append(result)

    return chapters


def main():
    loop = asyncio.get_event_loop()
    chapters = loop.run_until_complete(run())
    print(len(chapters))


if __name__ == '__main__':
    main()