From 6ba3197c9864071e3770669c844ddd0f0840b578 Mon Sep 17 00:00:00 2001 From: petrCher <88943157+petrCher@users.noreply.github.com> Date: Mon, 9 Mar 2026 23:21:06 +0300 Subject: [PATCH 1/7] fix union data --- dags/ods/user/user_to_back.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dags/ods/user/user_to_back.py b/dags/ods/user/user_to_back.py index 8d31767..15203c4 100644 --- a/dags/ods/user/user_to_back.py +++ b/dags/ods/user/user_to_back.py @@ -55,6 +55,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) phone_record = cursor.fetchone() + phone_record = phone_record.split(",")[0] result["phone_number"] = str(phone_record[0] if phone_record else "") logging.info(f"Took phone_number for {user_id} from dwh database") cursor.execute( @@ -66,6 +67,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) card_record = cursor.fetchone() + card_record = card_record.split(",")[0] result["card_number"] = str(card_record[0] if card_record else "") logging.info(f"Took card_number for {user_id} from dwh database") cursor.execute( @@ -77,6 +79,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) fullname_record = cursor.fetchone() + fullname_record = fullname_record.split(",")[0] result["full_name"] = str(fullname_record[0] if fullname_record else "") logging.info(f"Took full_name for {user_id} from dwh database") cursor.execute( @@ -88,6 +91,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) fullname_eng_record = cursor.fetchone() + fullname_eng_record = fullname_eng_record.split(",")[0] result["full_name_eng"] = str(fullname_eng_record[0] if fullname_eng_record else "") logging.info(f"Took full_name_eng for {user_id} from dwh database") cursor.execute( @@ -99,6 +103,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) birthday_record = cursor.fetchone() + birthday_record = birthday_record.split(",")[0] result["birthday"] = str(birthday_record[0] if birthday_record else "") logging.info(f"Took birthday for {user_id} from dwh database") cursor.execute( @@ -110,6 +115,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) faculty_record = cursor.fetchone() + faculty_record = faculty_record.split(",")[0] result["faculty"] = str(faculty_record[0] if faculty_record else "") logging.info(f"Took faculty for {user_id} from dwh database") cursor.execute( @@ -121,6 +127,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) faculty_eng_record = cursor.fetchone() + faculty_eng_record = faculty_eng_record.split(",")[0] result["faculty_eng"] = str(faculty_eng_record[0] if faculty_eng_record else "") logging.info(f"Took faculty_eng for {user_id} from dwh database") cursor.execute( @@ -132,6 +139,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) level_record = cursor.fetchone() + level_record = level_record.split(",")[0] result["education_level"] = str(level_record[0] if level_record else "") logging.info(f"Took education_level for {user_id} from dwh database") cursor.execute( @@ -143,6 +151,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) photo_record = cursor.fetchone() + photo_record = photo_record.split(",")[0] result["photo"] = str(photo_record[0] if photo_record else "") logging.info(f"Took photo for {user_id} from dwh database") return result From 8d6506f5cb9f61ef932cc97e42994f902152cec2 Mon Sep 17 00:00:00 2001 From: petrCher <88943157+petrCher@users.noreply.github.com> Date: Mon, 9 Mar 2026 23:53:44 +0300 Subject: [PATCH 2/7] fix data --- dags/ods/user/user_to_back.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/dags/ods/user/user_to_back.py b/dags/ods/user/user_to_back.py index 15203c4..60ebc82 100644 --- a/dags/ods/user/user_to_back.py +++ b/dags/ods/user/user_to_back.py @@ -55,8 +55,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) phone_record = cursor.fetchone() - phone_record = phone_record.split(",")[0] - result["phone_number"] = str(phone_record[0] if phone_record else "") + result["phone_number"] = (str(phone_record[0] if phone_record else "")).split(",")[0] logging.info(f"Took phone_number for {user_id} from dwh database") cursor.execute( f""" @@ -67,8 +66,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) card_record = cursor.fetchone() - card_record = card_record.split(",")[0] - result["card_number"] = str(card_record[0] if card_record else "") + result["card_number"] = (str(card_record[0] if card_record else "")).split(",")[0] logging.info(f"Took card_number for {user_id} from dwh database") cursor.execute( f""" @@ -79,8 +77,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) fullname_record = cursor.fetchone() - fullname_record = fullname_record.split(",")[0] - result["full_name"] = str(fullname_record[0] if fullname_record else "") + result["full_name"] = (str(fullname_record[0] if fullname_record else "")).split(",")[0] logging.info(f"Took full_name for {user_id} from dwh database") cursor.execute( f""" @@ -91,8 +88,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) fullname_eng_record = cursor.fetchone() - fullname_eng_record = fullname_eng_record.split(",")[0] - result["full_name_eng"] = str(fullname_eng_record[0] if fullname_eng_record else "") + result["full_name_eng"] = (str(fullname_eng_record[0] if fullname_eng_record else "")).split(",")[0] logging.info(f"Took full_name_eng for {user_id} from dwh database") cursor.execute( f""" @@ -103,8 +99,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) birthday_record = cursor.fetchone() - birthday_record = birthday_record.split(",")[0] - result["birthday"] = str(birthday_record[0] if birthday_record else "") + result["birthday"] = (str(birthday_record[0] if birthday_record else "")).split(",")[0] logging.info(f"Took birthday for {user_id} from dwh database") cursor.execute( f""" @@ -115,8 +110,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) faculty_record = cursor.fetchone() - faculty_record = faculty_record.split(",")[0] - result["faculty"] = str(faculty_record[0] if faculty_record else "") + result["faculty"] = (str(faculty_record[0] if faculty_record else "")).split(",")[0] logging.info(f"Took faculty for {user_id} from dwh database") cursor.execute( f""" @@ -127,8 +121,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) faculty_eng_record = cursor.fetchone() - faculty_eng_record = faculty_eng_record.split(",")[0] - result["faculty_eng"] = str(faculty_eng_record[0] if faculty_eng_record else "") + result["faculty_eng"] = (str(faculty_eng_record[0] if faculty_eng_record else "")).split(",")[0] logging.info(f"Took faculty_eng for {user_id} from dwh database") cursor.execute( f""" @@ -139,8 +132,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) level_record = cursor.fetchone() - level_record = level_record.split(",")[0] - result["education_level"] = str(level_record[0] if level_record else "") + result["education_level"] = (str(level_record[0] if level_record else "")).split(",")[0] logging.info(f"Took education_level for {user_id} from dwh database") cursor.execute( f""" @@ -151,14 +143,13 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) photo_record = cursor.fetchone() - photo_record = photo_record.split(",")[0] - result["photo"] = str(photo_record[0] if photo_record else "") + result["photo"] = (str(photo_record[0] if photo_record else "")).split(",")[0] logging.info(f"Took photo for {user_id} from dwh database") return result except Exception as e: logging.error( - f"Error ocured while collecting phone number and card_number for user {user_id} from dwh db: {str(e)}" + f"Error ocured while collecting union member data for user {user_id} from dwh db: {str(e)}" ) return result From da15358d556b6ab0a13401d4e502e0689e909bc4 Mon Sep 17 00:00:00 2001 From: petrCher <88943157+petrCher@users.noreply.github.com> Date: Tue, 10 Mar 2026 13:31:28 +0300 Subject: [PATCH 3/7] black fix --- dags/ods/user/user_to_back.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dags/ods/user/user_to_back.py b/dags/ods/user/user_to_back.py index 60ebc82..da6379a 100644 --- a/dags/ods/user/user_to_back.py +++ b/dags/ods/user/user_to_back.py @@ -148,9 +148,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: return result except Exception as e: - logging.error( - f"Error ocured while collecting union member data for user {user_id} from dwh db: {str(e)}" - ) + logging.error(f"Error ocured while collecting union member data for user {user_id} from dwh db: {str(e)}") return result From 5900521586e8b425060252707313ed8b625e00f3 Mon Sep 17 00:00:00 2001 From: petrCher <88943157+petrCher@users.noreply.github.com> Date: Tue, 10 Mar 2026 18:42:06 +0300 Subject: [PATCH 4/7] fix date --- dags/ods/user/info.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/ods/user/info.sql b/dags/ods/user/info.sql index bb1f45c..8e2e332 100644 --- a/dags/ods/user/info.sql +++ b/dags/ods/user/info.sql @@ -1305,4 +1305,4 @@ on conflict(card_id, user_id) do update set modified = CURRENT_TIMESTAMP; -drop table temp_combined_data; \ No newline at end of file +drop table temp_combined_data; From 14f0c090679889880cabea364f5cdb7dfbf37489 Mon Sep 17 00:00:00 2001 From: petrCher <88943157+petrCher@users.noreply.github.com> Date: Tue, 10 Mar 2026 18:43:34 +0300 Subject: [PATCH 5/7] fix --- dags/ods/user/info.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dags/ods/user/info.sql b/dags/ods/user/info.sql index 8e2e332..0bea8ef 100644 --- a/dags/ods/user/info.sql +++ b/dags/ods/user/info.sql @@ -147,12 +147,12 @@ select COALESCE( CASE WHEN ud.birthday ~ '^\d{2}\.\d{2}\.\d{4}$' THEN - TO_TIMESTAMP(ud.birthday, 'DD.MM.YYYY') + ud.birthday ELSE NULL END, CASE WHEN um.birthday ~ '^\d{4}-\d{2}-\d{2}' THEN - um.birthday::TIMESTAMP + um.birthday ELSE NULL END ) AS birthday, @@ -1305,4 +1305,4 @@ on conflict(card_id, user_id) do update set modified = CURRENT_TIMESTAMP; -drop table temp_combined_data; +drop table temp_combined_data; \ No newline at end of file From a7d0dfb4c3b95f617193175c6edf805801199e01 Mon Sep 17 00:00:00 2001 From: petrCher <88943157+petrCher@users.noreply.github.com> Date: Tue, 10 Mar 2026 22:24:48 +0300 Subject: [PATCH 6/7] fixed data --- dags/ods/user/info.sql | 10 ++++------ dags/ods/user/user_to_back.py | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/dags/ods/user/info.sql b/dags/ods/user/info.sql index 0bea8ef..8e135aa 100644 --- a/dags/ods/user/info.sql +++ b/dags/ods/user/info.sql @@ -218,12 +218,10 @@ select when um.full_name is not null and ud.full_names is not null and um.full_name != ANY(ud.full_names) then CONCAT(um.source, ', ', array_to_string(ud.full_name_sources, ', ')) when um.full_name is not null then um.source when ud.full_names is not null then array_to_string(ud.full_name_sources, ', ') - end as full_name_source, - CASE - WHEN um.full_name_eng IS NOT NULL AND ud.full_names IS NOT NULL AND um.full_name_eng != ANY(ud.full_names) THEN array_to_string(array_append(ud.full_names, um.full_name_eng), ', ') - WHEN um.full_name_eng IS NOT NULL THEN um.full_name_eng - WHEN ud.full_names IS NOT NULL THEN array_to_string(ud.full_names, ', ') - END as full_name_eng, + end as full_name_source, + case + when um.full_name_eng is not null then um.full_name_eng + end as full_name_eng, ud.git_hub_username as git_hub_username, ud.git_hub_username_source as git_hub_username_source, ud.home_phone_number as home_phone_number, diff --git a/dags/ods/user/user_to_back.py b/dags/ods/user/user_to_back.py index da6379a..4fc017c 100644 --- a/dags/ods/user/user_to_back.py +++ b/dags/ods/user/user_to_back.py @@ -77,7 +77,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict: """ ) fullname_record = cursor.fetchone() - result["full_name"] = (str(fullname_record[0] if fullname_record else "")).split(",")[0] + result["full_name"] = (str(fullname_record[0] if fullname_record else "")).split(",")[-1] logging.info(f"Took full_name for {user_id} from dwh database") cursor.execute( f""" @@ -171,7 +171,7 @@ def post_union_members_to_backend(union_members_ids: list): {"category": "Контакты", "param": "Номер телефона", "value": str(info['phone_number'])}, {"category": "Учетные данные", "param": "Номер профсоюзного билета", "value": str(info['card_number'])}, {"category": "Личная информация", "param": "Полное имя", "value": str(info['full_name'])}, - {"category": "Личная информация", "param": "Полное имя", "value": str(info['full_name_eng'])}, + {"category": "Личная информация", "param": "Full name", "value": str(info['full_name_eng'])}, {"category": "Личная информация", "param": "Дата рождения", "value": str(info['birthday'])}, {"category": "Учёба", "param": "Факультет", "value": str(info['faculty'])}, {"category": "Учёба", "param": "Факультет", "value": str(info['faculty_eng'])}, From 0f89c5199db44d5dc99e3a8c5773bfeabc0b7d5c Mon Sep 17 00:00:00 2001 From: petrCher <88943157+petrCher@users.noreply.github.com> Date: Tue, 10 Mar 2026 22:31:46 +0300 Subject: [PATCH 7/7] fixes --- dags/ods/user/info.sql | 8 +++----- dags/ods/user/user_to_back.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/dags/ods/user/info.sql b/dags/ods/user/info.sql index 8e135aa..4d51c9d 100644 --- a/dags/ods/user/info.sql +++ b/dags/ods/user/info.sql @@ -811,18 +811,16 @@ insert into "ODS_USERDATA".full_name_eng( select full_name_eng, user_id, - full_name_source, + 'union_member', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, False from ( select distinct full_name_eng, - user_id::integer as user_id, - full_name_source + user_id::integer as user_id from temp_combined_data - where full_name_eng is not null and trim(full_name_eng) != '' and full_name_source is not null and trim(full_name_source) != '' - and student_id is not null and trim(student_id) != '' + where full_name_eng is not null and trim(full_name_eng) != '' and student_id is not null and trim(student_id) != '' ) dedup on conflict(user_id, name) do update set name = EXCLUDED.name, diff --git a/dags/ods/user/user_to_back.py b/dags/ods/user/user_to_back.py index 4fc017c..56b4ca7 100644 --- a/dags/ods/user/user_to_back.py +++ b/dags/ods/user/user_to_back.py @@ -174,7 +174,7 @@ def post_union_members_to_backend(union_members_ids: list): {"category": "Личная информация", "param": "Full name", "value": str(info['full_name_eng'])}, {"category": "Личная информация", "param": "Дата рождения", "value": str(info['birthday'])}, {"category": "Учёба", "param": "Факультет", "value": str(info['faculty'])}, - {"category": "Учёба", "param": "Факультет", "value": str(info['faculty_eng'])}, + {"category": "Учёба", "param": "Faculty", "value": str(info['faculty_eng'])}, {"category": "Учёба", "param": "Ступень обучения", "value": str(info['education_level'])}, { "category": "Учёба",