diff --git a/CHANGELOG.md b/CHANGELOG.md index ec85d368..21d213d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Unreleased ## New features +- Added `fct_student_disability`, a new fact table unifying student disabilities from ed org associations and special education program associations into a single model. Includes `is_program` to distinguish the two grains, `k_student_program` for joining to program association fact tables, and extensible boolean designation columns via `xwalk_disability_designations`. +- Breaking change: New xwalk is required, `xwalk_disability_designations`, to pivot disability designation descriptors into boolean indicator columns in `fct_student_disability`. Expected columns: `disability_designation_descriptor` (the Ed-Fi descriptor value to match) and `indicator_name` (the name of the resulting boolean column). If the seed is empty, the designation columns are omitted and the model builds without them. ## Under the hood ## Fixes diff --git a/models/build/edfi_3/students/_edfi_3__students.yml b/models/build/edfi_3/students/_edfi_3__students.yml index ec10948b..70d5f13c 100644 --- a/models/build/edfi_3/students/_edfi_3__students.yml +++ b/models/build/edfi_3/students/_edfi_3__students.yml @@ -84,4 +84,10 @@ models: - name: bld_ef3__student_program__migrant_education config: tags: ['migrant_education'] - enabled: "{{ var('src:program:migrant_education:enabled', True) }}" \ No newline at end of file + enabled: "{{ var('src:program:migrant_education:enabled', True) }}" + - name: bld_ef3__student__disabilities + config: + tags: ['special_ed'] + - name: bld_ef3__student__wide_disability_designations + config: + tags: ['special_ed'] \ No newline at end of file diff --git a/models/build/edfi_3/students/bld_ef3__student__disabilities.sql b/models/build/edfi_3/students/bld_ef3__student__disabilities.sql new file mode 100644 index 00000000..6b9a9dab --- /dev/null +++ b/models/build/edfi_3/students/bld_ef3__student__disabilities.sql @@ -0,0 +1,49 @@ +-- Define all optional disability models here. +{% set stage_disability_relations = [] %} + +-- Ed Org Disabilities +{% do stage_disability_relations.append(ref('stg_ef3__stu_ed_org__disabilities')) %} + +-- Special Education +{% if var('src:program:special_ed:enabled', True) %} + {% do stage_disability_relations.append(ref('stg_ef3__stu_spec_ed__disabilities')) %} +{% endif %} + +with stacked as ( + {{ dbt_utils.union_relations( + relations=stage_disability_relations + ) }} +), +formatted as ( + select + -- Generated here so bld_ef3__student__wide_disability_designations can carry it forward, + -- allowing fct_student_disability to join on a single key rather than a null-safe multi-column join. + {{ dbt_utils.generate_surrogate_key([ + 'tenant_code', + 'school_year', + 'k_student', + 'k_lea', + 'k_school', + 'k_program', + 'program_enroll_begin_date', + 'disability_type', + ]) }} as k_student_disability, + tenant_code, + school_year, + k_student, + k_lea, + k_school, + k_program, + k_student_program, + program_enroll_begin_date, + program_enroll_end_date, + disability_type, + disability_source_type, + disability_diagnosis, + order_of_disability, + v_designations, + -- if k_program is populated, this is a program disability; otherwise it is an ed org disability. + k_program is not null as is_program + from stacked +) +select * from formatted diff --git a/models/build/edfi_3/students/bld_ef3__student__wide_disability_designations.sql b/models/build/edfi_3/students/bld_ef3__student__wide_disability_designations.sql new file mode 100644 index 00000000..f0919017 --- /dev/null +++ b/models/build/edfi_3/students/bld_ef3__student__wide_disability_designations.sql @@ -0,0 +1,47 @@ +with student_disabilities as ( + select * from {{ ref('bld_ef3__student__disabilities') }} +), +xwalk_disability_designations as ( + select * from {{ ref('xwalk_disability_designations') }} +), +flattened as ( + select + k_student_disability, + tenant_code, + school_year, + k_student, + k_lea, + k_school, + k_program, + k_student_program, + is_program, + disability_type, + {{ edu_edfi_source.extract_descriptor('designation.value:disabilityDesignationDescriptor::string') }} as disability_designation + from student_disabilities + {{ edu_edfi_source.json_flatten('v_designations', 'designation', outer=true) }} +), +pivoted as ( + select + k_student_disability, + tenant_code, + school_year, + k_student, + k_lea, + k_school, + k_program, + k_student_program, + is_program, + disability_type + {%- if not is_empty_model('xwalk_disability_designations') -%}, + {{ ea_pivot( + column='indicator_name', + values=dbt_utils.get_column_values(ref('xwalk_disability_designations'), 'indicator_name'), + cast='boolean', + ) }} + {%- endif %} + from flattened + left join xwalk_disability_designations + on flattened.disability_designation = xwalk_disability_designations.disability_designation_descriptor + group by all +) +select * from pivoted diff --git a/models/core_warehouse/fct_student_disability.sql b/models/core_warehouse/fct_student_disability.sql new file mode 100644 index 00000000..7f16e4e3 --- /dev/null +++ b/models/core_warehouse/fct_student_disability.sql @@ -0,0 +1,54 @@ +{{ + config( + post_hook=[ + "alter table {{ this }} alter column k_student_disability set not null", + "alter table {{ this }} alter column k_student set not null", + "alter table {{ this }} alter column disability_type set not null", + "alter table {{ this }} add primary key (k_student_disability)", + "alter table {{ this }} add constraint fk_{{ this.name }}_student foreign key (k_student) references {{ ref('dim_student') }}", + "alter table {{ this }} add constraint fk_{{ this.name }}_program foreign key (k_program) references {{ ref('dim_program') }}", + ] + ) +}} + +with student_disabilities as ( + select * from {{ ref('bld_ef3__student__disabilities') }} +), +dim_student as ( + select * from {{ ref('dim_student') }} +), +student_disability_designations as ( + select * from {{ ref('bld_ef3__student__wide_disability_designations') }} +), +formatted as ( + select + student_disabilities.k_student_disability, + student_disabilities.k_student, + dim_student.k_student_xyear, + student_disabilities.k_lea, + student_disabilities.k_school, + student_disabilities.k_program, + student_disabilities.k_student_program, + student_disabilities.tenant_code, + student_disabilities.school_year, + student_disabilities.is_program, + student_disabilities.program_enroll_begin_date, + student_disabilities.program_enroll_end_date, + student_disabilities.disability_type, + student_disabilities.disability_source_type, + student_disabilities.disability_diagnosis, + student_disabilities.order_of_disability, + -- Pivot disability designation descriptors into boolean columns. + {{ accordion_columns( + source_table='bld_ef3__student__wide_disability_designations', + exclude_columns=['k_student_disability', 'tenant_code', 'api_year', 'school_year', 'k_student', 'k_lea', 'k_school', 'k_program', 'k_student_program', 'is_program', 'disability_type'], + source_alias='student_disability_designations', + add_trailing_comma=false + ) }} + from student_disabilities + join dim_student + on dim_student.k_student = student_disabilities.k_student + left join student_disability_designations + on student_disabilities.k_student_disability = student_disability_designations.k_student_disability +) +select * from formatted diff --git a/models/core_warehouse/fct_student_disability.yml b/models/core_warehouse/fct_student_disability.yml new file mode 100644 index 00000000..40b4021c --- /dev/null +++ b/models/core_warehouse/fct_student_disability.yml @@ -0,0 +1,105 @@ +version: 2 + +models: + - name: fct_student_disability + description: > + ##### Overview: + This fact table provides student disabilities, which could be assigned from multiple sources. + It references any or all of the following models: + - [stg_ef3__stu_ed_org__disabilities](#!/model/model.edu_edfi_source.stg_ef3__stu_ed_org__disabilities) + - [stg_ef3__stu_spec_ed__disabilities](#!/model/model.edu_edfi_source.stg_ef3__stu_spec_ed__disabilities) + + ##### Primary Key: + `k_student_disability`: one record per student, year, ed org or program enrollment, and disability type. + + ##### Important business rules: + - This table is at two different grains since both Ed Orgs and Special Ed Programs can have disabilities listed. + Use `is_program` to distinguish: `false` means an ed org disability, `true` means a program disability. + - `program_enroll_begin_date`, `k_program` and `k_student_program` are null for ed org rows. When joining to + `fct_student_{program_type}_program_association`, join on `k_student_program` (see example query below). + + ##### Example Use Cases: + 1. Join Disabilities to Student Special Education data to find students' Special Ed disabilities + they received as part of that program: + + ``` + SELECT + assoc.k_student, + assoc.school_year, + assoc.k_program, + dim_program.program_type, + dim_program.program_id, + dim_program.program_name, + assoc.program_enroll_begin_date, + assoc.program_enroll_end_date, + dis.disability_type, + dis.disability_source_type, + dis.disability_diagnosis, + dis.order_of_disability + FROM analytics.prod_wh.fct_student_special_education_program_association assoc + JOIN analytics.prod_wh.dim_program + ON assoc.k_program = dim_program.k_program + -- left join because some programs may not have disabilities + LEFT JOIN analytics.prod_wh.fct_student_disability dis + ON assoc.k_student_program = dis.k_student_program; + ``` + + + config: + tags: ['special_ed'] + + constraints: + - type: primary_key + columns: [k_student_disability] + - type: foreign_key + name: fk__fct_student_disability__k_student + columns: [k_student] + to: ref('dim_student') + to_columns: [k_student] + - type: foreign_key + name: fk__fct_student_disability__k_program + columns: [k_program] + to: ref('dim_program') + to_columns: [k_program] + + columns: + - name: k_student_disability + description: > + Surrogate primary key. Generated from `tenant_code`, `school_year`, `k_student`, `k_lea`, + `k_school`, `k_program`, `program_enroll_begin_date`, and `disability_type`. + - name: k_student + description: Unique identifier for the student-year. Foreign key reference to [dim_student](#!/model/model.edu_wh.dim_student). + - name: k_student_xyear + description: Defining key for the student, which is consistent across years. + - name: k_lea + description: Unique key identifying the Local Education Agency (LEA) associated with this disability record. + - name: k_school + description: Unique key identifying the school associated with this disability record. + - name: k_program + description: > + Unique identifier for the program associated with this disability. Foreign key reference to + [dim_program](#!/model/model.edu_wh.dim_program). Null for ed org disability rows. + - name: k_student_program + description: > + Surrogate key for the student's program enrollment. Use this to join to + `fct_student_{program_type}_program_association` tables. Null for ed org disability rows. + - name: tenant_code + description: The tenant associated with the record. + - name: school_year + description: > + School year specified by Spring year. + e.g. the 2021-2022 year would be 2022. + - name: is_program + description: Whether the disability is sourced from a program association (true) or an ed org association (false). + - name: program_enroll_begin_date + description: The date the student began enrollment in the program associated with this disability. Null for ed org disability rows. + - name: program_enroll_end_date + description: The date the student's enrollment in the associated program ended, if applicable. Null for ed org disability rows. + - name: disability_type + description: A disability category that describes the student's impairment (e.g. Autism, Intellectual Disability). + - name: disability_source_type + description: The source that provided the disability determination (e.g. Medical, Self). + - name: disability_diagnosis + description: A more specific description of the disability diagnosis, if provided. + - name: order_of_disability + description: The ordinal ranking when a student has multiple disabilities listed, where 1 indicates the primary disability.