Upgrade to Django 3.0

- upgrades
  - Django 3.0.14
  - django-jsonfield 1.4.1 (from 1.2.0)
  - django-oauth-toolkit 1.4.1 (from 1.1.3)
    - Stopping here because later versions have changes to the
      underlying model to support OpenID Connect.  Presumably this can
      be dealt with via a migration in our project.
  - django-guid 2.2.1 (from 2.2.0)
  - django-debug-toolbar 3.2.4 (from 1.11.1)
  - python3-saml 1.13.0 (from 1.9.0)
  - xmlsec 1.3.12 (from 1.3.3)

- Remove our project's use of django.utils.six in favor of directly
  using six, in awx.sso.fields.

- Temporarily monkey patch six back in as django.utils.six, since
  django-jsonfield makes use of that import, and is no longer being
  updated.  Hopefully we can do away with this dependency with the new
  generalized JSONField brought in with Django 3.1.

- Force a json decoder to be used with all instances of JSONField
  brought in by django-jsonfield.  This deals with the 'cast to text'
  problem noted previously in our UPGRADE_BLOCKERS.

- Remove the validate_uris validator from the OAuth2Application in
  migration 0025, per the UPGRADE_BLOCKERS, and remove that note.

- Update the TEMPLATES setting to satisfy Django Debug Toolbar.  It
  requires at least one entry that has APP_DIRS=True, and as near as I
  can tell our custom OPTIONS.loaders setting was effectively doing
  the same thing as Django's own machinery if this setting is set.

requirements/README.md

@@ -58,7 +58,7 @@ Make sure to delete the old tarball if it is an upgrade.
 Anything pinned in `*.in` files involves additional manual work in
 order to upgrade. Some information related to that work is outlined here.
 
-### django
+### Django
 
 For any upgrade of Django, it must be confirmed that
 we don't regress on FIPS support before merging.
@@ -90,13 +90,10 @@ that we have the latest version
 
 ### django-oauth-toolkit
 
-Version 1.2.0 of this project has a bug that error when revoking tokens.
-This is fixed in the master branch but is not yet released.
-
-When upgrading past 1.2.0 in the future, the `0025` migration needs to be
-edited, just like the old migration was edited in the project:
-https://github.com/jazzband/django-oauth-toolkit/commit/96538876d0d7ea0319ba5286f9bde842a906e1c5
-The field can simply have the validator method `validate_uris` removed.
+Versions later than 1.4.1 throw an error about id_token_id, due to the
+OpenID Connect work that was done in
+https://github.com/jazzband/django-oauth-toolkit/pull/915.  This may
+be fixable by creating a migration on our end?
 
 ### azure-keyvault
 
@@ -117,7 +114,7 @@ https://github.com/adamchainz/django-jsonfield/pull/14
 
 This breaks a very large amount of AWX code that assumes these fields
 are returned as dicts. Upgrading this library will require a refactor
-to accomidate this change.
+to accommodate this change.
 
 ### pip and setuptools
 

requirements/requirements.in

@@ -9,14 +9,14 @@ cryptography>=3.2
 Cython<3 # Since the bump to PyYAML 5.4.1 this is now a mandatory dep
 daphne
 distro
-django==2.2.24  # see UPGRADE BLOCKERs
+django==3.0.14  # see UPGRADE BLOCKERs
 django-auth-ldap
 django-cors-headers>=3.5.0
 django-crum
 django-extensions>=2.2.9  # https://github.com/ansible/awx/pull/6441
-django-guid==2.2.0  # pinned to match Django 2.2
-django-jsonfield==1.2.0  # see UPGRADE BLOCKERs
-django-oauth-toolkit==1.1.3  # see UPGRADE BLOCKERs
+django-guid==2.2.1  # see https://pypi.org/project/django-guid/ for supported versions
+django-jsonfield==1.4.1
+django-oauth-toolkit==1.4.1
 django-polymorphic
 django-pglocks
 django-qsstats-magic
@@ -40,7 +40,7 @@ psycopg2
 psutil
 pygerduty
 pyparsing
-python3-saml
+python3-saml==1.13.0
 python-dsv-sdk
 python-tss-sdk==1.0.0
 python-ldap>=3.3.1 # https://github.com/python-ldap/python-ldap/issues/270

requirements/requirements.txt

@@ -14,6 +14,7 @@ asgiref==3.2.5
     #   channels
     #   channels-redis
     #   daphne
+    #   django
 async-timeout==3.0.1
     # via
     #   aiohttp
@@ -80,13 +81,12 @@ dataclasses==0.6
 defusedxml==0.6.0
     # via
     #   python3-openid
-    #   python3-saml
     #   social-auth-core
 dictdiffer==0.8.1
     # via openshift
 distro==1.5.0
     # via -r /awx_devel/requirements/requirements.in
-django==2.2.24
+django==3.0.14
     # via
     #   -r /awx_devel/requirements/requirements.in
     #   channels
@@ -107,11 +107,11 @@ django-crum==0.7.5
     # via -r /awx_devel/requirements/requirements.in
 django-extensions==2.2.9
     # via -r /awx_devel/requirements/requirements.in
-django-guid==2.2.0
+django-guid==2.2.1
     # via -r /awx_devel/requirements/requirements.in
-django-jsonfield==1.2.0
+django-jsonfield==1.4.1
     # via -r /awx_devel/requirements/requirements.in
-django-oauth-toolkit==1.1.3
+django-oauth-toolkit==1.4.1
     # via -r /awx_devel/requirements/requirements.in
 django-pglocks==1.0.4
     # via -r /awx_devel/requirements/requirements.in
@@ -159,7 +159,7 @@ incremental==17.5.0
     # via twisted
 irc==18.0.0
     # via -r /awx_devel/requirements/requirements.in
-isodate==0.6.0
+isodate==0.6.1
     # via
     #   msrest
     #   python3-saml
@@ -180,7 +180,7 @@ jaraco.text==3.2.0
     # via
     #   irc
     #   jaraco.collections
-jinja2==2.11.2
+jinja2==3.0.3
     # via
     #   -r /awx_devel/requirements/requirements.in
     #   openshift
@@ -192,11 +192,13 @@ kubernetes==11.0.0
     # via openshift
 lockfile==0.12.2
     # via python-daemon
-lxml==4.6.3
-    # via xmlsec
+lxml==4.7.0
+    # via
+    #   python3-saml
+    #   xmlsec
 markdown==3.2.1
     # via -r /awx_devel/requirements/requirements.in
-markupsafe==1.1.1
+markupsafe==2.0.1
     # via jinja2
 more-itertools==8.2.0
     # via
@@ -232,8 +234,6 @@ pexpect==4.7.0
     # via
     #   -r /awx_devel/requirements/requirements.in
     #   ansible-runner
-pkgconfig==1.5.1
-    # via xmlsec
 prometheus-client==0.7.1
     # via -r /awx_devel/requirements/requirements.in
 psutil==5.8.0
@@ -293,7 +293,7 @@ python-tss-sdk==1.0.0
     # via -r /awx_devel/requirements/requirements.in
 python3-openid==3.1.0
     # via social-auth-core
-python3-saml==1.9.0
+python3-saml==1.13.0
     # via -r /awx_devel/requirements/requirements.in
 pytz==2019.3
     # via
@@ -336,8 +336,6 @@ rsa==4.7.2
     # via google-auth
 ruamel.yaml==0.16.10
     # via openshift
-ruamel.yaml.clib==0.2.0
-    # via ruamel.yaml
 schedule==0.6.0
     # via -r /awx_devel/requirements/requirements.in
 service-identity==18.1.0
@@ -348,6 +346,7 @@ six==1.14.0
     #   automat
     #   cryptography
     #   django-extensions
+    #   django-jsonfield
     #   django-pglocks
     #   google-auth
     #   isodate
@@ -407,7 +406,7 @@ websocket-client==0.57.0
     # via kubernetes
 wheel==0.36.2
     # via -r /awx_devel/requirements/requirements.in
-xmlsec==1.3.3
+xmlsec==1.3.12
     # via python3-saml
 yarl==1.4.2
     # via aiohttp

requirements/requirements_dev.txt

@@ -1,4 +1,4 @@
-django-debug-toolbar==1.11.1
+django-debug-toolbar==3.2.4
 django-rest-swagger
 # pprofile - re-add once https://github.com/vpelletier/pprofile/issues/41 is addressed
 ipython==7.21.0

requirements/updater.sh

@@ -18,7 +18,8 @@ generate_requirements() {
   # shellcheck disable=SC1090
   source ${venv}/bin/activate
 
-  ${venv}/bin/python3 -m pip install -U pip pip-tools
+  # FIXME: https://github.com/jazzband/pip-tools/issues/1558
+  ${venv}/bin/python3 -m pip install -U 'pip<22.0' pip-tools
 
   ${pip_compile} "${requirements_in}" "${requirements_git}" --output-file requirements.txt
   # consider the git requirements for purposes of resolving deps