diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..d0e2fa2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,37 @@ +name: release +on: + push: + tags: + - 'v*.*.*' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: build + shell: bash + run: | + python -m pip install --upgrade wheel gevent + python setup.py sdist bdist_wheel --universal + - name: Release PyPI + shell: bash + env: + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + pip install --upgrade twine + twine upload dist/* + + - name: Release GitHub + uses: softprops/action-gh-release@v1 + with: + files: "dist/*" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..3c7b97a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,20 @@ +on: [ push, pull_request ] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install gevent requests pytest nose + - name: Test with pytest + run: | + pytest tests.py diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c4f010d..0000000 --- a/.travis.yml +++ /dev/null @@ -1,27 +0,0 @@ -dist: xenial -language: python - - -python: - - "2.7" - - "3.6" - - "3.7" - - -install: - - pip install -r requirements.txt - - pip install pytest - -script: - - pytest tests.py - - -deploy: - provider: pypi - distributions: "sdist bdist_wheel" - user: "spyoungtech" - password: - secure: "QtuuH0X/A/iQI23MxvqsnxUy63XD5awJHDkeQNmUDIGGQqIox2DTYKoc6x354I5wpqprtODQRYRqIsA9+2cpRcF49Ft50cvi3cmuoeozkID3ybQyLHCIcJ4CKt6X+h2LFbrgqyyBcny7tKQlYr4/nsjeQegPblnJ6OTljJgJyE0=" - on: - tags: true - python: 3.6 \ No newline at end of file diff --git a/README.rst b/README.rst index e586c5f..1254e9e 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,14 @@ Requests easily. |version| |pyversions| -**Note**: You should probably use `requests-threads `_ or `requests-futures `_ instead. + +Installation +------------ + +Installation is easy with pip:: + + $ pip install grequests + ✨🍰✨ Usage @@ -34,16 +41,23 @@ Create a set of unsent Requests: >>> rs = (grequests.get(u) for u in urls) -Send them all at the same time: +Send them all at the same time using ``map``: .. code-block:: python >>> grequests.map(rs) [, , , , None, ] -Optionally, in the event of a timeout or any other exception during the connection of -the request, you can add an exception handler that will be called with the request and -exception inside the main thread: + +The HTTP verb methods in ``grequests`` (e.g., ``grequests.get``, ``grequests.post``, etc.) accept all the same keyword arguments as in the ``requests`` library. + +Error Handling +^^^^^^^^^^^^^^ + +To handle timeouts or any other exception during the connection of +the request, you can add an optional exception handler that will be called with the request and +exception inside the main thread. The value returned by your exception handler will be used in the result list returned by ``map``. + .. code-block:: python @@ -59,15 +73,53 @@ exception inside the main thread: Request failed [None, None, ] -For some speed/performance gains, you may also want to use `imap` instead of `map`. `imap` returns a generator of responses. Order of these responses does not map to the order of the requests you send out. The API for `imap` is equivalent to the API for `map`. -Installation ------------- +imap +^^^^ + +For some speed/performance gains, you may also want to use ``imap`` instead of ``map``. ``imap`` returns a generator of responses. Order of these responses does not map to the order of the requests you send out. The API for ``imap`` is equivalent to the API for ``map``. You can also adjust the ``size`` argument to ``map`` or ``imap`` to increase the gevent pool size. + + +.. code-block:: python + + for resp in grequests.imap(reqs, size=10): + print(resp) + + +There is also an enumerated version of ``imap``, ``imap_enumerated`` which yields the index of the request from the original request list and its associated response. However, unlike ``imap``, failed requests and exception handler results that return ``None`` will also be yielded (whereas in ``imap`` they are ignored). Aditionally, the ``requests`` parameter for ``imap_enumerated`` must be a sequence. Like in ``imap``, the order in which requests are sent and received should still be considered arbitrary. + +.. code-block:: python + + >>> rs = [grequests.get(f'https://httpbin.org/status/{code}') for code in range(200, 206)] + >>> for index, response in grequests.imap_enumerated(rs, size=5): + ... print(index, response) + 1 + 0 + 4 + 2 + 5 + 3 + +gevent - when things go wrong +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Because ``grequests`` leverages ``gevent`` (which in turn uses monkeypatching for enabling concurrency), you will often need to make sure ``grequests`` is imported before other libraries, especially ``requests``, to avoid problems. See `grequests gevent issues `_ for additional information. + + +.. code-block:: python + + # GOOD + import grequests + import requests + + # BAD + import requests + import grequests + + + -Installation is easy with pip:: - $ pip install grequests - ✨🍰✨ .. |version| image:: https://img.shields.io/pypi/v/grequests.svg?colorB=blue @@ -75,3 +127,5 @@ Installation is easy with pip:: .. |pyversions| image:: https://img.shields.io/pypi/pyversions/grequests.svg? :target: https://pypi.org/project/grequests/ + + diff --git a/grequests.py b/grequests.py index f4ec5c7..acf1933 100755 --- a/grequests.py +++ b/grequests.py @@ -163,3 +163,51 @@ def send(r): yield ex_result pool.join() + + +def imap_enumerated(requests, stream=False, size=2, exception_handler=None): + """ + Like imap, but yields tuple of original request index and response object + + Unlike imap, failed results and responses from exception handlers that return None are not ignored. Instead, a + tuple of (index, None) is yielded. Additionally, the ``requests`` parameter must be a sequence of Request objects + (generators or other non-sequence iterables are not allowed) + + The index is merely the original index of the original request in the requests list and does NOT provide any + indication of the order in which requests or responses are sent or received. Responses are still in arbitrary order. + + :: + >>> rs = [grequests.get(f'https://httpbin.org/status/{i}') for i in range(200, 206)] + >>> for index, response in grequests.imap_enumerated(rs, size=5): + ... print(index, response) + 1 + 0 + 4 + 2 + 5 + 3 + + + :param requests: a sequence of Request objects. + :param stream: If True, the content will not be downloaded immediately. + :param size: Specifies the number of requests to make at a time. default is 2 + :param exception_handler: Callback function, called when exception occurred. Params: Request, Exception + """ + + pool = Pool(size) + + def send(r): + return r._index, r.send(stream=stream) + + requests = list(requests) + for index, req in enumerate(requests): + req._index = index + + for index, request in pool.imap_unordered(send, requests): + if request.response is not None: + yield index, request.response + elif exception_handler: + ex_result = exception_handler(request, request.exception) + yield index, ex_result + else: + yield index, None diff --git a/setup.py b/setup.py index 9122abd..0343841 100644 --- a/setup.py +++ b/setup.py @@ -33,8 +33,8 @@ setup( name='grequests', - version='0.6.0', - url='https://github.com/kennethreitz/grequests', + version='0.7.0', + url='https://github.com/spyoungtech/grequests', license='BSD', author='Kenneth Reitz', author_email='me@kennethreitz.com', diff --git a/tests.py b/tests.py index 2fda406..8d500ce 100644 --- a/tests.py +++ b/tests.py @@ -165,7 +165,7 @@ def test_imap_timeout_no_exception_handler(self): out.append(r) except Timeout: pass - self.assertEquals(out, []) + self.assertEqual(out, []) def test_imap_timeout_exception_handler_no_return(self): """ @@ -177,7 +177,7 @@ def exception_handler(request, exception): out = [] for r in grequests.imap(reqs, exception_handler=exception_handler): out.append(r) - self.assertEquals(out, []) + self.assertEqual(out, []) def test_imap_timeout_exception_handler_returns_value(self): @@ -190,7 +190,7 @@ def exception_handler(request, exception): out = [] for r in grequests.imap(reqs, exception_handler=exception_handler): out.append(r) - self.assertEquals(out, ['a value']) + self.assertEqual(out, ['a value']) def test_map_timeout_exception(self): class ExceptionHandler: