From 14b5f63bd824785a9afeac4588057ae27b51e179 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 13 Apr 2020 17:27:29 -0400 Subject: [PATCH] Use the new PageCache to store and reuse Page.get results --- awxkit/awxkit/api/pages/api.py | 33 +++++++++++------------ awxkit/awxkit/api/pages/page.py | 45 ++++++++++++++++++++++++++------ awxkit/awxkit/api/pages/roles.py | 14 +++++----- 3 files changed, 60 insertions(+), 32 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 887420b8d7..c7e90c4f28 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -78,14 +78,13 @@ class ApiV2(base.Base): for key in options: if key not in asset.related: continue - try: - # FIXME: use caching by url - natural_key = asset.related[key].get().get_natural_key() - except exc.Forbidden: - log.warning("This foreign key cannot be read: %s", asset.related[key]) - return None + + related_endpoint = self._cache.get_page(asset.related[key]) + if related_endpoint is None: + return None # This foreign key is unreadable + natural_key = related_endpoint.get_natural_key(self._cache) if natural_key is None: - return None # This is an unresolvable foreign key + return None # This foreign key has unresolvable dependencies fields[key] = natural_key related = {} @@ -106,16 +105,13 @@ class ApiV2(base.Base): else: continue - try: - # FIXME: use caching by url - data = rel.get(all_pages=True) - except exc.Forbidden: - log.warning("These related objects cannot be read: %s", related_endpoint) + data = self._cache.get_page(related_endpoint) + if data is None: continue if 'results' in data: results = ( - x.get_natural_key() if by_natural_key else self._serialize_asset(x, related_options) + x.get_natural_key(self._cache) if by_natural_key else self._serialize_asset(x, related_options) for x in data.results ) related[key] = [x for x in results if x is not None] @@ -125,7 +121,7 @@ class ApiV2(base.Base): if related: fields['related'] = related - natural_key = asset.get_natural_key() + natural_key = asset.get_natural_key(self._cache) if natural_key is None: return None fields['natural_key'] = natural_key @@ -144,7 +140,7 @@ class ApiV2(base.Base): pk = pk_or_name(self, resource, value) # TODO: decide whether to support multiple results = endpoint.get(id=pk).results else: - results = endpoint.get(all_pages=True).results + results = self._cache.get_page(endpoint).results assets = (self._serialize_asset(asset, options) for asset in results) return [asset for asset in assets if asset is not None] @@ -174,7 +170,7 @@ class ApiV2(base.Base): yield page_resource[page_cls] def _register_page(self, page): - natural_key = freeze(page.get_natural_key()) + natural_key = freeze(page.get_natural_key(self._cache)) # FIXME: we need to keep a reference for the case where we # don't have a natural key, so we can delete if natural_key is not None: @@ -189,7 +185,7 @@ class ApiV2(base.Base): if options is None: return - results = endpoint.get(all_pages=True).results + results = self._cache.get_page(endpoint).results for pg in results: self._register_page(pg) @@ -237,6 +233,7 @@ class ApiV2(base.Base): page = endpoint.post(post_data) else: page = page.put(post_data) + # FIXME: created pages need to be put in the cache except exc.Common: log.exception("post_data: %r", post_data) raise @@ -274,7 +271,7 @@ class ApiV2(base.Base): for item in related_set: data = {key: value for key, value in item.items() if key not in ('natural_key', 'related')} - endpoint.post(data) + endpoint.post(data) # FIXME: add the page to the cache # FIXME: deal with objects that themselves have relateds, e.g. WFJT Nodes # FIXME: deal with pruning existing relations that do not match the import set diff --git a/awxkit/awxkit/api/pages/page.py b/awxkit/awxkit/api/pages/page.py index 94943ec6b6..b4f0eaf0b9 100644 --- a/awxkit/awxkit/api/pages/page.py +++ b/awxkit/awxkit/api/pages/page.py @@ -317,9 +317,12 @@ class Page(object): page_cls = get_registered_page(endpoint) return page_cls(self.connection, endpoint=endpoint).get(**kw) - def get_natural_key(self): + def get_natural_key(self, cache=None): warn = "This object does not have a natural key: %s" + if cache is None: + cache = PageCache() + if not getattr(self, 'NATURAL_KEY', None): log.warning(warn, getattr(self, 'endpoint', '')) return None @@ -327,12 +330,10 @@ class Page(object): natural_key = {} for key in self.NATURAL_KEY: if key in self.related: - try: - # FIXME: use caching by url - natural_key[key] = self.related[key].get().get_natural_key() - except exc.Forbidden: - log.warning("This foreign key cannot be read: %s", self.related[key]) + related_endpoint = cache.get_page(self.related[key]) + if related_endpoint is None: return None + natural_key[key] = related_endpoint.get_natural_key(cache=cache) elif key in self: natural_key[key] = self[key] if not natural_key: @@ -401,7 +402,7 @@ class PageList(object): def create(self, *a, **kw): return self.__item_class__(self.connection).create(*a, **kw) - def get_natural_key(self): + def get_natural_key(self, cache=None): log.warning("This object does not have a natural key: %s", getattr(self, 'endpoint', '')) return None @@ -536,19 +537,47 @@ class TentativePage(str): class PageCache(object): def __init__(self): self.options = {} + self.pages_by_url = {} def get_options(self, page): - url = page.url if isinstance(page, Page) else str(page) + url = page.endpoint if isinstance(page, Page) else str(page) if url in self.options: return self.options[url] try: options = page.options() except exc.Common: + log.error("This endpoint raised an error: %s", url) return self.options.setdefault(url, None) warning = options.r.headers.get('Warning', '') if '299' in warning and 'deprecated' in warning: + log.warning("This endpoint is deprecated: %s", url) return self.options.setdefault(url, None) return self.options.setdefault(url, options) + + def set_page(self, page): + self.pages_by_url[page.endpoint] = page + if 'results' in page: + for p in page.results: + self.set_page(p) + return page + + def get_page(self, page): + url = page.endpoint if isinstance(page, Page) else str(page) + if url in self.pages_by_url: + return self.pages_by_url[url] + + try: + page = page.get(all_pages=True) + except exc.Common: + log.error("This endpoint raised an error: %s", url) + return self.pages_by_url.setdefault(url, None) + + warning = page.r.headers.get('Warning', '') + if '299' in warning and 'deprecated' in warning: + log.warning("This endpoint is deprecated: %s", url) + return self.pages_by_url.setdefault(url, None) + + return self.set_page(page) diff --git a/awxkit/awxkit/api/pages/roles.py b/awxkit/awxkit/api/pages/roles.py index d93de4cad1..530dfc36e9 100644 --- a/awxkit/awxkit/api/pages/roles.py +++ b/awxkit/awxkit/api/pages/roles.py @@ -9,18 +9,20 @@ class Role(base.Base): NATURAL_KEY = ('name',) - def get_natural_key(self): - natural_key = super(Role, self).get_natural_key() + def get_natural_key(self, cache=None): + if cache is None: + cache = page.PageCache() + + natural_key = super(Role, self).get_natural_key(cache=cache) related_objs = [ related for name, related in self.related.items() if name not in ('users', 'teams') ] if related_objs: - try: - # FIXME: use caching by url - natural_key['content_object'] = related_objs[0].get().get_natural_key() - except exc.Forbidden: + related_endpoint = cache.get_page(related_objs[0]) + if related_endpoint is None: return None + natural_key['content_object'] = related_endpoint.get_natural_key(cache=cache) return natural_key