Skip to content

fix union data #342

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
petrCher merged 7 commits into from
Mar 10, 2026
Merged

fix union data #342

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6ba3197
fix union data
petrCher Mar 9, 2026
8d6506f
fix data
petrCher Mar 9, 2026
da15358
black fix
petrCher Mar 10, 2026
5900521
fix date
petrCher Mar 10, 2026
14f0c09
fix
petrCher Mar 10, 2026
a7d0dfb
fixed data
petrCher Mar 10, 2026
0f89c51
fixes
petrCher Mar 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 9 additions & 13 deletions dags/ods/user/info.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,12 +147,12 @@ select
COALESCE(
CASE
WHEN ud.birthday ~ '^\d{2}\.\d{2}\.\d{4}$' THEN
TO_TIMESTAMP(ud.birthday, 'DD.MM.YYYY')
ud.birthday
ELSE NULL
END,
CASE
WHEN um.birthday ~ '^\d{4}-\d{2}-\d{2}' THEN
um.birthday::TIMESTAMP
um.birthday
ELSE NULL
END
) AS birthday,
Expand Down Expand Up @@ -218,12 +218,10 @@ select
when um.full_name is not null and ud.full_names is not null and um.full_name != ANY(ud.full_names) then CONCAT(um.source, ', ', array_to_string(ud.full_name_sources, ', '))
when um.full_name is not null then um.source
when ud.full_names is not null then array_to_string(ud.full_name_sources, ', ')
end as full_name_source,
CASE
WHEN um.full_name_eng IS NOT NULL AND ud.full_names IS NOT NULL AND um.full_name_eng != ANY(ud.full_names) THEN array_to_string(array_append(ud.full_names, um.full_name_eng), ', ')
WHEN um.full_name_eng IS NOT NULL THEN um.full_name_eng
WHEN ud.full_names IS NOT NULL THEN array_to_string(ud.full_names, ', ')
END as full_name_eng,
end as full_name_source,
case
when um.full_name_eng is not null then um.full_name_eng
end as full_name_eng,
ud.git_hub_username as git_hub_username,
ud.git_hub_username_source as git_hub_username_source,
ud.home_phone_number as home_phone_number,
Expand Down Expand Up @@ -813,18 +811,16 @@ insert into "ODS_USERDATA".full_name_eng(
select
full_name_eng,
user_id,
full_name_source,
'union_member',
CURRENT_TIMESTAMP,
CURRENT_TIMESTAMP,
False
from (
select distinct
full_name_eng,
user_id::integer as user_id,
full_name_source
user_id::integer as user_id
from temp_combined_data
where full_name_eng is not null and trim(full_name_eng) != '' and full_name_source is not null and trim(full_name_source) != ''
and student_id is not null and trim(student_id) != ''
where full_name_eng is not null and trim(full_name_eng) != '' and student_id is not null and trim(student_id) != ''
) dedup
on conflict(user_id, name) do update set
name = EXCLUDED.name,
Expand Down
26 changes: 12 additions & 14 deletions dags/ods/user/user_to_back.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict:
"""
)
phone_record = cursor.fetchone()
result["phone_number"] = str(phone_record[0] if phone_record else "")
result["phone_number"] = (str(phone_record[0] if phone_record else "")).split(",")[0]
logging.info(f"Took phone_number for {user_id} from dwh database")
cursor.execute(
f"""
Expand All @@ -66,7 +66,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict:
"""
)
card_record = cursor.fetchone()
result["card_number"] = str(card_record[0] if card_record else "")
result["card_number"] = (str(card_record[0] if card_record else "")).split(",")[0]
logging.info(f"Took card_number for {user_id} from dwh database")
cursor.execute(
f"""
Expand All @@ -77,7 +77,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict:
"""
)
fullname_record = cursor.fetchone()
result["full_name"] = str(fullname_record[0] if fullname_record else "")
result["full_name"] = (str(fullname_record[0] if fullname_record else "")).split(",")[-1]
logging.info(f"Took full_name for {user_id} from dwh database")
cursor.execute(
f"""
Expand All @@ -88,7 +88,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict:
"""
)
fullname_eng_record = cursor.fetchone()
result["full_name_eng"] = str(fullname_eng_record[0] if fullname_eng_record else "")
result["full_name_eng"] = (str(fullname_eng_record[0] if fullname_eng_record else "")).split(",")[0]
logging.info(f"Took full_name_eng for {user_id} from dwh database")
cursor.execute(
f"""
Expand All @@ -99,7 +99,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict:
"""
)
birthday_record = cursor.fetchone()
result["birthday"] = str(birthday_record[0] if birthday_record else "")
result["birthday"] = (str(birthday_record[0] if birthday_record else "")).split(",")[0]
logging.info(f"Took birthday for {user_id} from dwh database")
cursor.execute(
f"""
Expand All @@ -110,7 +110,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict:
"""
)
faculty_record = cursor.fetchone()
result["faculty"] = str(faculty_record[0] if faculty_record else "")
result["faculty"] = (str(faculty_record[0] if faculty_record else "")).split(",")[0]
logging.info(f"Took faculty for {user_id} from dwh database")
cursor.execute(
f"""
Expand All @@ -121,7 +121,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict:
"""
)
faculty_eng_record = cursor.fetchone()
result["faculty_eng"] = str(faculty_eng_record[0] if faculty_eng_record else "")
result["faculty_eng"] = (str(faculty_eng_record[0] if faculty_eng_record else "")).split(",")[0]
logging.info(f"Took faculty_eng for {user_id} from dwh database")
cursor.execute(
f"""
Expand All @@ -132,7 +132,7 @@ def get_phone_number_by_user_ids(user_id: int) -> dict:
"""
)
level_record = cursor.fetchone()
result["education_level"] = str(level_record[0] if level_record else "")
result["education_level"] = (str(level_record[0] if level_record else "")).split(",")[0]
logging.info(f"Took education_level for {user_id} from dwh database")
cursor.execute(
f"""
Expand All @@ -143,14 +143,12 @@ def get_phone_number_by_user_ids(user_id: int) -> dict:
"""
)
photo_record = cursor.fetchone()
result["photo"] = str(photo_record[0] if photo_record else "")
result["photo"] = (str(photo_record[0] if photo_record else "")).split(",")[0]
logging.info(f"Took photo for {user_id} from dwh database")
return result

except Exception as e:
logging.error(
f"Error ocured while collecting phone number and card_number for user {user_id} from dwh db: {str(e)}"
)
logging.error(f"Error ocured while collecting union member data for user {user_id} from dwh db: {str(e)}")
return result


Expand All @@ -173,10 +171,10 @@ def post_union_members_to_backend(union_members_ids: list):
{"category": "Контакты", "param": "Номер телефона", "value": str(info['phone_number'])},
{"category": "Учетные данные", "param": "Номер профсоюзного билета", "value": str(info['card_number'])},
{"category": "Личная информация", "param": "Полное имя", "value": str(info['full_name'])},
{"category": "Личная информация", "param": "Полное имя", "value": str(info['full_name_eng'])},
{"category": "Личная информация", "param": "Full name", "value": str(info['full_name_eng'])},
{"category": "Личная информация", "param": "Дата рождения", "value": str(info['birthday'])},
{"category": "Учёба", "param": "Факультет", "value": str(info['faculty'])},
{"category": "Учёба", "param": "Факультет", "value": str(info['faculty_eng'])},
{"category": "Учёба", "param": "Faculty", "value": str(info['faculty_eng'])},
{"category": "Учёба", "param": "Ступень обучения", "value": str(info['education_level'])},
{
"category": "Учёба",
Expand Down