diff --git a/dags/ods/user/info.sql b/dags/ods/user/info.sql index bb1f45c..4d51c9d 100644 --- a/dags/ods/user/info.sql +++ b/dags/ods/user/info.sql @@ -147,12 +147,12 @@ select COALESCE( CASE WHEN ud.birthday ~ '^\d{2}\.\d{2}\.\d{4}$' THEN - TO_TIMESTAMP(ud.birthday, 'DD.MM.YYYY') + ud.birthday ELSE NULL END, CASE WHEN um.birthday ~ '^\d{4}-\d{2}-\d{2}' THEN - um.birthday::TIMESTAMP + um.birthday ELSE NULL END ) AS birthday, @@ -218,12 +218,10 @@ select when um.full_name is not null and ud.full_names is not null and um.full_name != ANY(ud.full_names) then CONCAT(um.source, ', ', array_to_string(ud.full_name_sources, ', ')) when um.full_name is not null then um.source when ud.full_names is not null then array_to_string(ud.full_name_sources, ', ') - end as full_name_source, - CASE - WHEN um.full_name_eng IS NOT NULL AND ud.full_names IS NOT NULL AND um.full_name_eng != ANY(ud.full_names) THEN array_to_string(array_append(ud.full_names, um.full_name_eng), ', ') - WHEN um.full_name_eng IS NOT NULL THEN um.full_name_eng - WHEN ud.full_names IS NOT NULL THEN array_to_string(ud.full_names, ', ') - END as full_name_eng, + end as full_name_source, + case + when um.full_name_eng is not null then um.full_name_eng + end as full_name_eng, ud.git_hub_username as git_hub_username, ud.git_hub_username_source as git_hub_username_source, ud.home_phone_number as home_phone_number, @@ -813,18 +811,16 @@ insert into "ODS_USERDATA".full_name_eng( select full_name_eng, user_id, - full_name_source, + 'union_member', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, False from ( select distinct full_name_eng, - user_id::integer as user_id, - full_name_source + user_id::integer as user_id from temp_combined_data - where full_name_eng is not null and trim(full_name_eng) != '' and full_name_source is not null and trim(full_name_source) != '' - and student_id is not null and trim(student_id) != '' + where full_name_eng is not null and trim(full_name_eng) != '' and student_id is not null and trim(student_id) != '' ) dedup on conflict(user_id, name) do update set name = EXCLUDED.name, diff --git a/dags/ods/user/user_to_back.py b/dags/ods/user/user_to_back.py index 8d31767..56b4ca7 100644 --- a/dags/ods/user/user_to_back.py +++ b/dags/ods/user/user_to_back.py @@ -55,7 +55,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) phone_record = cursor.fetchone() - result["phone_number"] = str(phone_record[0] if phone_record else "") + result["phone_number"] = (str(phone_record[0] if phone_record else "")).split(",")[0] logging.info(f"Took phone_number for {user_id} from dwh database") cursor.execute( f""" @@ -66,7 +66,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) card_record = cursor.fetchone() - result["card_number"] = str(card_record[0] if card_record else "") + result["card_number"] = (str(card_record[0] if card_record else "")).split(",")[0] logging.info(f"Took card_number for {user_id} from dwh database") cursor.execute( f""" @@ -77,7 +77,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) fullname_record = cursor.fetchone() - result["full_name"] = str(fullname_record[0] if fullname_record else "") + result["full_name"] = (str(fullname_record[0] if fullname_record else "")).split(",")[-1] logging.info(f"Took full_name for {user_id} from dwh database") cursor.execute( f""" @@ -88,7 +88,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) fullname_eng_record = cursor.fetchone() - result["full_name_eng"] = str(fullname_eng_record[0] if fullname_eng_record else "") + result["full_name_eng"] = (str(fullname_eng_record[0] if fullname_eng_record else "")).split(",")[0] logging.info(f"Took full_name_eng for {user_id} from dwh database") cursor.execute( f""" @@ -99,7 +99,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) birthday_record = cursor.fetchone() - result["birthday"] = str(birthday_record[0] if birthday_record else "") + result["birthday"] = (str(birthday_record[0] if birthday_record else "")).split(",")[0] logging.info(f"Took birthday for {user_id} from dwh database") cursor.execute( f""" @@ -110,7 +110,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) faculty_record = cursor.fetchone() - result["faculty"] = str(faculty_record[0] if faculty_record else "") + result["faculty"] = (str(faculty_record[0] if faculty_record else "")).split(",")[0] logging.info(f"Took faculty for {user_id} from dwh database") cursor.execute( f""" @@ -121,7 +121,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) faculty_eng_record = cursor.fetchone() - result["faculty_eng"] = str(faculty_eng_record[0] if faculty_eng_record else "") + result["faculty_eng"] = (str(faculty_eng_record[0] if faculty_eng_record else "")).split(",")[0] logging.info(f"Took faculty_eng for {user_id} from dwh database") cursor.execute( f""" @@ -132,7 +132,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) level_record = cursor.fetchone() - result["education_level"] = str(level_record[0] if level_record else "") + result["education_level"] = (str(level_record[0] if level_record else "")).split(",")[0] logging.info(f"Took education_level for {user_id} from dwh database") cursor.execute( f""" @@ -143,14 +143,12 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) photo_record = cursor.fetchone() - result["photo"] = str(photo_record[0] if photo_record else "") + result["photo"] = (str(photo_record[0] if photo_record else "")).split(",")[0] logging.info(f"Took photo for {user_id} from dwh database") return result except Exception as e: - logging.error( - f"Error ocured while collecting phone number and card_number for user {user_id} from dwh db: {str(e)}" - ) + logging.error(f"Error ocured while collecting union member data for user {user_id} from dwh db: {str(e)}") return result @@ -173,10 +171,10 @@ def post_union_members_to_backend(union_members_ids: list): {"category": "Контакты", "param": "Номер телефона", "value": str(info['phone_number'])}, {"category": "Учетные данные", "param": "Номер профсоюзного билета", "value": str(info['card_number'])}, {"category": "Личная информация", "param": "Полное имя", "value": str(info['full_name'])}, - {"category": "Личная информация", "param": "Полное имя", "value": str(info['full_name_eng'])}, + {"category": "Личная информация", "param": "Full name", "value": str(info['full_name_eng'])}, {"category": "Личная информация", "param": "Дата рождения", "value": str(info['birthday'])}, {"category": "Учёба", "param": "Факультет", "value": str(info['faculty'])}, - {"category": "Учёба", "param": "Факультет", "value": str(info['faculty_eng'])}, + {"category": "Учёба", "param": "Faculty", "value": str(info['faculty_eng'])}, {"category": "Учёба", "param": "Ступень обучения", "value": str(info['education_level'])}, { "category": "Учёба",