Al cerrarMi servidor MySQL desaparece después de miuna consulta SQL sobre la tabla cacia user en mi ETL obtengo un error:
(venv) C:\Users\antoi\Documents\Programming\Work\data-tools>python -m etl.main
2021-0607-2906 10:5912:37:15.814133134728 - Connecting to database hozana_data...
2021-0607-2906 10:5912:37:15.822142140731 - Connecting to archive database hozana_archive...
2021-0607-2906 1012:5937:3815.046134372728 - Start ETL main process
2021-0607-2906 1012:5937:3815.046134372728 - `users` table:
2021-0607-2906 1012:5937:3815.046134372728 - Hashing column `users`.`email:`
table_name: users
c.execute("SELECT * FROM users"): 0
done.
2021-0607-2906 1012:5937:3815.054091374727 - Hashing column `users`.`email_notification:`Traceback`
table_name: users
(2006, '')
2021-07-06 12:37:15.375740 - Hashing column `users`.`last_name:`
table_name: users
(2006, '')
2021-07-06 12:37:15.376728 - `messages` table:
Traceback (most recent call last):
File "C:\Users\antoi\Documents\Programming\Work\data-tools\etl\task\anonymization.py", line 1771, in hash_columnmain
c.execute(""""""UPDATE messages SET payload = '' WHERE type = 'text_message'""")
File "C:\Users\antoi\Documents\Programming\Work\data-tools\venv\lib\site-packages\MySQLdb\cursors.py", line 183, in execute
while self.nextset():
File "C:\Users\antoi\Documents\Programming\Work\data-tools\venv\lib\site-packages\MySQLdb\cursors.py", line 137, in nextset
nr = db.next_result()
MySQLdb._exceptions.OperationalError: (2006, '')
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\antoi\AppData\Local\Programs\Python\Python39\lib\runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "C:\Users\antoi\AppData\Local\Programs\Python\Python39\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "C:\Users\antoi\Documents\Programming\Work\data-tools\etl\main.py", line 52, in <module>
main()
File "C:\Users\antoi\Documents\Programming\Work\data-tools\etl\main.py", line 24, in main
anonymization.main()
File "C:\Users\antoi\Documents\Programming\Work\data-tools\etl\task\anonymization.py", line 5971, in main
hash_column('users', 'email_notification', 'user_id', True)
File "C:\Users\antoi\Documents\Programming\Work\data-tools\etl\task\anonymizationc.py", lineexecute("""UPDATE 50,messages inSET hash_column
payload = '' WHERE print('.',type end='',= flush=True'text_message'""")
File "C:\Users\antoi\Documents\Programming\Work\data-tools\venv\lib\site-packages\MySQLdb\connections.py", line 239, in __exit__
self.close()
MySQLdb._exceptions.OperationalError: (2006, '')
Aqui esta el ETL. Este script fue hecho cuando la biblioteca era 1.3.7 y estoy tratando de cambiar a 2.0.3codigo:
def hash_column(table_name, column_name, pk_name, email_mode=False):
# Executed a query that will overwrite a column with an hashed version of its content.
# With email mode, the domain name will be kept:
# [email protected] -> 379ac32fe8f576c4c63b17cd576e6c40c7dcd[...][email protected]
try:
info('Hashing column `{table_name}`.`{column_name}:`'
.format(table_name=table_name, column_name=column_name), end='', flush=True)
with db as c:
# Execute the update in batches, to avoid "Lock wait timeout exceeded"
print("\ntable_name: ", table_name)
c = c.cursor()
print("c.execute(\"""SELECT * FROM users\"""): ", c.execute("SELECT * FROM users"))
c.execute("""
SELECT {pk_name} as row_id
FROM {table_name}
WHERE {column_name} IS NOT NULL AND {column_name} NOT LIKE 'hash_%'
ORDER BY row_id ASC
""".format(table_name=table_name, column_name=column_name, pk_name=pk_name))
ids = []
for row in c.fetchall():
row_id, = row
# cast to string, to avoid "expected str instance, int found" on join
ids.append(str(row_id))
ids_chunks = chunks(list(ids), 50000)
for ids in ids_chunks:
expression = """
CONCAT(
SHA2(SUBSTRING_INDEX(LOWER({column_name}), '@', 1), 512),
'@',
SUBSTRING_INDEX(LOWER({column_name}), '@', -1)
)
""".format(column_name=column_name) if email_mode \
else 'SHA2(LOWER({column_name}), 512)'.format(column_name=column_name)
print("c.execute(\"""SELECT COUNT(*) FROM users\"""): ",
c.execute("""SELECT COUNT(*) FROM {table_name}""").format(table_name=table_name))
c.execute("""
UPDATE `{table_name}`
SET `{column_name}` = CONCAT('hash_', {expression})
WHERE {pk_name} IN ({ids})
""".format(
table_name=table_name, column_name=column_name, expression=expression, pk_name=pk_name,
ids=','.join(ids)
))
print('.', end='', flush=True)
print(' done.')
except MySQLdb.OperationalError as e:
print(e)