zware/awx
1
0
Fork 0
mirror of https://github.com/ZwareBear/awx.git synced 2026-05-13 15:58:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

move the partition data migration to be a post-upgrade async process

Browse Source 
this copies the approach we took with the bigint migration
This commit is contained in:
Ryan Petrello
2021-02-25 14:12:26 -05:00
committed by Jim Ladd
parent 67046513ae
commit c7ab3ea86e
5 changed files with 45 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
awx/main/tasks.py
+14 -13
View File
@@ -81,6 +81,7 @@ from awx.main.models import (
AdHocCommandEvent,
SystemJobEvent,
build_safe_env,
migrate_events_to_partitions
)
from awx.main.constants import ACTIVE_STATES
from awx.main.exceptions import AwxTaskError, PostRunError
@@ -173,6 +174,12 @@ def dispatch_startup():
cluster_node_heartbeat()
Metrics().clear_values()
# at process startup, detect the need to migrate old event records to
# partitions; at *some point* in the future, once certain versions of AWX
# and Tower fall out of use/support, we can probably just _assume_ that
# everybody has moved to partitions, and remove this code entirely
migrate_events_to_partitions()
# Update Tower's rsyslog.conf file based on loggins settings in the db
reconfigure_rsyslog()
@@ -684,22 +691,16 @@ def update_host_smart_inventory_memberships():
@task(queue=get_local_queuename)
def migrate_legacy_event_data(tblname):
#
# NOTE: this function is not actually in use anymore,
# but has been intentionally kept for historical purposes,
# and to serve as an illustration if we ever need to perform
# bulk modification/migration of event data in the future.
#
if 'event' not in tblname:
return
with advisory_lock(f'bigint_migration_{tblname}', wait=False) as acquired:
with advisory_lock(f'partition_migration_{tblname}', wait=False) as acquired:
if acquired is False:
return
chunk = settings.JOB_EVENT_MIGRATION_CHUNK_SIZE
def _remaining():
try:
cursor.execute(f'SELECT MAX(id) FROM _old_{tblname};')
cursor.execute(f'SELECT MAX(id) FROM _unpartitioned_{tblname};')
return cursor.fetchone()[0]
except ProgrammingError:
# the table is gone (migration is unnecessary)
@@ -709,19 +710,19 @@ def migrate_legacy_event_data(tblname):
total_rows = _remaining()
while total_rows:
with transaction.atomic():
cursor.execute(f'INSERT INTO {tblname} SELECT * FROM _old_{tblname} ORDER BY id DESC LIMIT {chunk} RETURNING id;')
cursor.execute(f'''INSERT INTO {tblname} SELECT *, '1970-01-01' as job_created FROM _unpartitioned_{tblname} ORDER BY id DESC LIMIT {chunk} RETURNING id;''')
last_insert_pk = cursor.fetchone()
if last_insert_pk is None:
# this means that the SELECT from the old table was
# empty, and there was nothing to insert (so we're done)
break
last_insert_pk = last_insert_pk[0]
cursor.execute(f'DELETE FROM _old_{tblname} WHERE id IN (SELECT id FROM _old_{tblname} ORDER BY id DESC LIMIT {chunk});')
logger.warn(f'migrated int -> bigint rows to {tblname} from _old_{tblname}; # ({last_insert_pk} rows remaining)')
cursor.execute(f'DELETE FROM _unpartitioned_{tblname} WHERE id IN (SELECT id FROM _unpartitioned_{tblname} ORDER BY id DESC LIMIT {chunk});')
logger.warn(f'migrated rows to partitioned {tblname} from _unpartitioned_{tblname}; # ({last_insert_pk} rows remaining)')
if _remaining() is None:
cursor.execute(f'DROP TABLE IF EXISTS _old_{tblname}')
logger.warn(f'{tblname} primary key migration to bigint has finished')
cursor.execute(f'DROP TABLE IF EXISTS _unpartitioned_{tblname}')
logger.warn(f'{tblname} migration to partitions has finished')
@task(queue=get_local_queuename)