diff --git a/src/pyff/api.py b/src/pyff/api.py index 87a2e266..37a3ca74 100644 --- a/src/pyff/api.py +++ b/src/pyff/api.py @@ -1,5 +1,6 @@ import importlib import threading +import traceback from collections.abc import Generator, Iterable, Mapping from datetime import datetime, timedelta from json import dumps @@ -265,54 +266,74 @@ def _d(x: Optional[str], do_split: bool = True) -> tuple[Optional[str], Optional else: q = new_path + # The pipeline result - often the whole selected metadata tree - only lives in _run_plumbings' frame. Pyramid keeps + # an error response's exception on the request, so an error must not reference that frame through its traceback: + # request, exception and frame would form a reference cycle, freed only by a full garbage collection (issue #283). try: - accepter = MediaAccept(accept) - for p in request.registry.plumbings: - state = { - entry: True, - 'headers': {'Content-Type': None}, - 'accept': accepter, - 'url': request.current_route_url(), - 'select': q, - 'match': match.lower() if match else match, - 'path': new_path, - 'stats': {}, - } - - r = p.process(request.registry.md, state=state, raise_exceptions=True, scheduler=request.registry.scheduler) - log.debug(f'Plumbing process result: {r}') - if r is None: - r = [] - - response = Response() - _headers = state.get('headers', {}) - response.headers.update(_headers) - ctype = _headers.get('Content-Type', None) - if not ctype: - r, t = _fmt(r, accepter) - ctype = t - - response.text = b2u(r) - response.size = len(r) - response.content_type = ctype - cache_ttl = int(state.get('cache', 0)) - response.expires = datetime.now() + timedelta(seconds=cache_ttl) - return response + response = _run_plumbings(request, entry, accept, q, match, new_path) + except exc.HTTPException as ex: + # An HTTP error from processing, eg 406 from content negotiation, is the response as is. Besides the traceback, + # __context__/__cause__ can reference the frame (an error raised while handling another one), so clear them too. + ex.__context__ = ex.__cause__ = None + raise ex.with_traceback(None) except ResourceException as ex: - import traceback - log.debug(traceback.format_exc()) log.warning(f'Exception from processing pipeline: {ex}') - raise exc.exception_response(409) + status = 409 except BaseException as ex: - import traceback - log.debug(traceback.format_exc()) log.error(f'Exception from processing pipeline: {ex}') - raise exc.exception_response(500) + status = 500 + else: + if response is None and request.method == 'GET': + raise exc.exception_response(404) + return response - if request.method == 'GET': - raise exc.exception_response(404) + # Raised outside the except clauses, so that it does not chain the original exception and its traceback. + raise exc.exception_response(status) + + +def _run_plumbings( + request: Request, entry: str, accept: str, q: Optional[str], match: Optional[str], new_path: Optional[str] +) -> Optional[Response]: + """ + Run the request through the plumbings and format the result for the accepted content type. + + :return: the response, or None if there are no plumbings + """ + accepter = MediaAccept(accept) + for p in request.registry.plumbings: + state = { + entry: True, + 'headers': {'Content-Type': None}, + 'accept': accepter, + 'url': request.current_route_url(), + 'select': q, + 'match': match.lower() if match else match, + 'path': new_path, + 'stats': {}, + } + + r = p.process(request.registry.md, state=state, raise_exceptions=True, scheduler=request.registry.scheduler) + log.debug(f'Plumbing process result: {r}') + if r is None: + r = [] + + response = Response() + _headers = state.get('headers', {}) + response.headers.update(_headers) + ctype = _headers.get('Content-Type', None) + if not ctype: + r, t = _fmt(r, accepter) + ctype = t + + response.text = b2u(r) + response.size = len(r) + response.content_type = ctype + cache_ttl = int(state.get('cache', 0)) + response.expires = datetime.now() + timedelta(seconds=cache_ttl) + return response + return None def webfinger_handler(request: Request) -> Response: diff --git a/src/pyff/test/test_md_api.py b/src/pyff/test/test_md_api.py index e3c59fc1..c1e5bed4 100644 --- a/src/pyff/test/test_md_api.py +++ b/src/pyff/test/test_md_api.py @@ -1,21 +1,26 @@ +import gc import json import os import shutil import tempfile import unittest +import weakref from datetime import datetime, timezone from types import SimpleNamespace +from unittest import mock from urllib.parse import quote as urlescape import pytest import requests from mako.lookup import TemplateLookup +from pyramid import httpexceptions as exc from pyramid.testing import DummyRequest from pyuppsala import etree from wsgi_intercept.interceptor import RequestsInterceptor, UrllibInterceptor from pyff.api import mkapp, search_handler from pyff.constants import config +from pyff.resource import ResourceException from pyff.samlmd import iter_entities from pyff.test import SignerTestCase from pyff.test.test_pipeline import PipeLineTest @@ -356,3 +361,110 @@ def test_entity_id_without_trailing_slash_is_not_found(self): with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url: assert requests.post(f'{url}/api/call/update').status_code == 200 assert self._entity_ids(url, '/entities/https:/sp.example.com/saml2/metadata') == set() + + +class _PipelineResult(list): + """A pipeline result that can be weakly referenced, to observe when it is freed""" + + +class PyFFAPITestErrorResponses(PipeLineTest): + """ + Errors while processing a request must have the right status, and must not keep the pipeline result - often + the whole selected metadata tree - alive after the request: cf. issue #283. + """ + + mdx = None + app = None + + @classmethod + def setUpClass(cls): + SignerTestCase.setUpClass() + config.local_copy_dir = tempfile.mkdtemp() + cls.mdx = tempfile.NamedTemporaryFile('w').name + with open(cls.mdx, "w") as fd: + fd.write( + f""" +- when update: + - load: + - {os.path.join(cls.datadir, 'metadata', 'test01.xml')} +- when request: + - select + - pipe: + - when accept application/xml: + - emit application/xml + - break +""" + ) + cls._app = mkapp(cls.mdx) + cls.app = lambda *args, **kwargs: cls._app + + @classmethod + def tearDownClass(cls): + SignerTestCase.tearDownClass() + if os.path.exists(cls.mdx): + os.unlink(cls.mdx) + if os.path.exists(config.local_copy_dir): + shutil.rmtree(config.local_copy_dir) + + def _request_with_pipeline_result(self, url, fail=False, error=None): + """ + GET /entities as text/plain with the pipeline replaced by one producing a weakly referenceable result, + then failing if ``fail`` (with a RuntimeError) or with the ``error`` factory's exception, raised while a + KeyError is being handled. The garbage collector is off, so only reference counting can free the result. + + :return: the HTTP status and whether the result was freed once the request was done + """ + refs = [] + + def process(*args, **kwargs): + result = _PipelineResult(['']) + refs.append(weakref.ref(result)) + if error is not None: + try: + raise KeyError("first failure") + except KeyError: + raise error() + if fail: + raise RuntimeError("pipeline failed") + return result + + gc.collect() + gc.disable() + try: + with mock.patch.object(self._app.registry.plumbings[0], 'process', process): + status = requests.get(f'{url}/entities', headers={'Accept': 'text/plain'}).status_code + return status, refs[0]() is None + finally: + gc.enable() + + def test_unacceptable_type_is_406(self): + """No pipeline branch and no format matches text/plain: that is 406 Not Acceptable, not a server error""" + with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url: + assert requests.post(f'{url}/api/call/update').status_code == 200 + r = requests.get(f'{url}/entities', headers={'Accept': 'text/plain'}) + assert r.status_code == 406 + + def test_406_frees_the_pipeline_result(self): + with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url: + status, freed = self._request_with_pipeline_result(url, fail=False) + assert status == 406 + assert freed, "the pipeline result outlived a 406 response" + + def test_500_frees_the_pipeline_result(self): + with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url: + status, freed = self._request_with_pipeline_result(url, fail=True) + assert status == 500 + assert freed, "the pipeline result outlived a 500 response" + + def test_http_error_raised_while_handling_another_frees_the_pipeline_result(self): + """An HTTP error chained to another exception must not keep the result alive through __context__""" + with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url: + status, freed = self._request_with_pipeline_result(url, error=exc.HTTPNotFound) + assert status == 404 + assert freed, "the pipeline result outlived a chained 404 response" + + def test_409_frees_the_pipeline_result(self): + with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url: + status, freed = self._request_with_pipeline_result(url, error=lambda: ResourceException("conflict")) + assert status == 409 + assert freed, "the pipeline result outlived a 409 response"