TDOE#230: Refactor Student Disability Models.

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Unreleased
 ## New features
+- Added `fct_student_disability`, a new fact table unifying student disabilities from ed org associations and special education program associations into a single model. Includes `is_program` to distinguish the two grains, `k_student_program` for joining to program association fact tables, and extensible boolean designation columns via `xwalk_disability_designations`.
+- Breaking change: New xwalk is required, `xwalk_disability_designations`, to pivot disability designation descriptors into boolean indicator columns in `fct_student_disability`. Expected columns: `disability_designation_descriptor` (the Ed-Fi descriptor value to match) and `indicator_name` (the name of the resulting boolean column). If the seed is empty, the designation columns are omitted and the model builds without them.
 ## Under the hood
 ## Fixes
 

models/build/edfi_3/students/_edfi_3__students.yml

@@ -84,4 +84,10 @@ models:
   - name: bld_ef3__student_program__migrant_education
     config:
       tags: ['migrant_education']
-      enabled: "{{ var('src:program:migrant_education:enabled', True) }}"
+      enabled: "{{ var('src:program:migrant_education:enabled', True) }}"
+  - name: bld_ef3__student__disabilities
+    config:
+      tags: ['special_ed']
+  - name: bld_ef3__student__wide_disability_designations
+    config:
+      tags: ['special_ed']

models/build/edfi_3/students/bld_ef3__student__disabilities.sql

@@ -0,0 +1,49 @@
+-- Define all optional disability models here.
+{% set stage_disability_relations = [] %}
+
+-- Ed Org Disabilities
+{% do stage_disability_relations.append(ref('stg_ef3__stu_ed_org__disabilities')) %}
+
+-- Special Education
+{% if var('src:program:special_ed:enabled', True) %}
+    {% do stage_disability_relations.append(ref('stg_ef3__stu_spec_ed__disabilities')) %}
+{% endif %}
+
+with stacked as (
+    {{ dbt_utils.union_relations(
+        relations=stage_disability_relations
+    ) }}
+),
+formatted as (
+    select
+        -- Generated here so bld_ef3__student__wide_disability_designations can carry it forward,
+        -- allowing fct_student_disability to join on a single key rather than a null-safe multi-column join.
+        {{ dbt_utils.generate_surrogate_key([
+            'tenant_code',
+            'school_year',
+            'k_student',
+            'k_lea',
+            'k_school',
+            'k_program',
+            'program_enroll_begin_date',
+            'disability_type',
+        ]) }} as k_student_disability,
+        tenant_code,
+        school_year,
+        k_student,
+        k_lea,
+        k_school,
+        k_program,
+        k_student_program,
+        program_enroll_begin_date,
+        program_enroll_end_date,
+        disability_type,
+        disability_source_type,
+        disability_diagnosis,
+        order_of_disability,
+        v_designations,
+        -- if k_program is populated, this is a program disability; otherwise it is an ed org disability.
+        k_program is not null as is_program
+    from stacked
+)
+select * from formatted

models/build/edfi_3/students/bld_ef3__student__wide_disability_designations.sql

@@ -0,0 +1,47 @@
+with student_disabilities as (
+    select * from {{ ref('bld_ef3__student__disabilities') }}
+),
+xwalk_disability_designations as (
+    select * from {{ ref('xwalk_disability_designations') }}
+),
+flattened as (
+    select
+        k_student_disability,
+        tenant_code,
+        school_year,
+        k_student,
+        k_lea,
+        k_school,
+        k_program,
+        k_student_program,
+        is_program,
+        disability_type,
+        {{ edu_edfi_source.extract_descriptor('designation.value:disabilityDesignationDescriptor::string') }} as disability_designation
+    from student_disabilities
+        {{ edu_edfi_source.json_flatten('v_designations', 'designation', outer=true) }}
+),
+pivoted as (
+    select
+        k_student_disability,
+        tenant_code,
+        school_year,
+        k_student,
+        k_lea,
+        k_school,
+        k_program,
+        k_student_program,
+        is_program,
+        disability_type
+        {%- if not is_empty_model('xwalk_disability_designations') -%},
+            {{ ea_pivot(
+                    column='indicator_name',
+                    values=dbt_utils.get_column_values(ref('xwalk_disability_designations'), 'indicator_name'),
+                    cast='boolean',
+            ) }}
+        {%- endif %}
+    from flattened
+    left join xwalk_disability_designations
+        on flattened.disability_designation = xwalk_disability_designations.disability_designation_descriptor
+    group by all
+)
+select * from pivoted

models/core_warehouse/fct_student_disability.sql

@@ -0,0 +1,54 @@
+{{
+  config(
+    post_hook=[
+        "alter table {{ this }} alter column k_student_disability set not null",
+        "alter table {{ this }} alter column k_student set not null",
+        "alter table {{ this }} alter column disability_type set not null",
+        "alter table {{ this }} add primary key (k_student_disability)",
+        "alter table {{ this }} add constraint fk_{{ this.name }}_student foreign key (k_student) references {{ ref('dim_student') }}",
+        "alter table {{ this }} add constraint fk_{{ this.name }}_program foreign key (k_program) references {{ ref('dim_program') }}",
+    ]
+  )
+}}
+
+with student_disabilities as (
+    select * from {{ ref('bld_ef3__student__disabilities') }}
+),
+dim_student as (
+    select * from {{ ref('dim_student') }}
+),
+student_disability_designations as (
+    select * from {{ ref('bld_ef3__student__wide_disability_designations') }}
+),
+formatted as (
+    select
+        student_disabilities.k_student_disability,
+        student_disabilities.k_student,
+        dim_student.k_student_xyear,
+        student_disabilities.k_lea,
+        student_disabilities.k_school,
+        student_disabilities.k_program,
+        student_disabilities.k_student_program,
+        student_disabilities.tenant_code,
+        student_disabilities.school_year,
+        student_disabilities.is_program,
+        student_disabilities.program_enroll_begin_date,
+        student_disabilities.program_enroll_end_date,
+        student_disabilities.disability_type,
+        student_disabilities.disability_source_type,
+        student_disabilities.disability_diagnosis,
+        student_disabilities.order_of_disability,
+        -- Pivot disability designation descriptors into boolean columns.
+        {{ accordion_columns(
+            source_table='bld_ef3__student__wide_disability_designations',
+            exclude_columns=['k_student_disability', 'tenant_code', 'api_year', 'school_year', 'k_student', 'k_lea', 'k_school', 'k_program', 'k_student_program', 'is_program', 'disability_type'],
+            source_alias='student_disability_designations',
+            add_trailing_comma=false
+        ) }}
+    from student_disabilities
+    join dim_student
+        on dim_student.k_student = student_disabilities.k_student
+    left join student_disability_designations
+        on student_disabilities.k_student_disability = student_disability_designations.k_student_disability
+)
+select * from formatted

models/core_warehouse/fct_student_disability.yml

@@ -0,0 +1,105 @@
+version: 2
+
+models:
+  - name: fct_student_disability
+    description: >
+     ##### Overview:
+       This fact table provides student disabilities, which could be assigned from multiple sources.
+       It references any or all of the following models:
+      - [stg_ef3__stu_ed_org__disabilities](#!/model/model.edu_edfi_source.stg_ef3__stu_ed_org__disabilities)
+      - [stg_ef3__stu_spec_ed__disabilities](#!/model/model.edu_edfi_source.stg_ef3__stu_spec_ed__disabilities)
+
+     ##### Primary Key:
+       `k_student_disability`: one record per student, year, ed org or program enrollment, and disability type.
+
+     ##### Important business rules:
+        - This table is at two different grains since both Ed Orgs and Special Ed Programs can have disabilities listed.
+          Use `is_program` to distinguish: `false` means an ed org disability, `true` means a program disability.
+        - `program_enroll_begin_date`, `k_program` and `k_student_program` are null for ed org rows. When joining to
+           `fct_student_{program_type}_program_association`, join on `k_student_program` (see example query below).
+
+     ##### Example Use Cases:
+       1. Join Disabilities to Student Special Education data to find students' Special Ed disabilities
+       they received as part of that program:
+
+        ```
+          SELECT
+            assoc.k_student,
+            assoc.school_year,
+            assoc.k_program,
+            dim_program.program_type,
+            dim_program.program_id,
+            dim_program.program_name,
+            assoc.program_enroll_begin_date,
+            assoc.program_enroll_end_date,
+            dis.disability_type,
+            dis.disability_source_type,
+            dis.disability_diagnosis,
+            dis.order_of_disability
+          FROM analytics.prod_wh.fct_student_special_education_program_association assoc
+          JOIN analytics.prod_wh.dim_program
+            ON assoc.k_program = dim_program.k_program
+          -- left join because some programs may not have disabilities
+          LEFT JOIN analytics.prod_wh.fct_student_disability dis
+            ON assoc.k_student_program = dis.k_student_program;
+        ```
+
+
+    config:
+      tags: ['special_ed']
+
+    constraints:
+      - type: primary_key
+        columns: [k_student_disability]
+      - type: foreign_key
+        name: fk__fct_student_disability__k_student
+        columns: [k_student]
+        to: ref('dim_student')
+        to_columns: [k_student]
+      - type: foreign_key
+        name: fk__fct_student_disability__k_program
+        columns: [k_program]
+        to: ref('dim_program')
+        to_columns: [k_program]
+
+    columns:
+      - name: k_student_disability
+        description: >
+          Surrogate primary key. Generated from `tenant_code`, `school_year`, `k_student`, `k_lea`,
+          `k_school`, `k_program`, `program_enroll_begin_date`, and `disability_type`.
+      - name: k_student
+        description: Unique identifier for the student-year. Foreign key reference to [dim_student](#!/model/model.edu_wh.dim_student).
+      - name: k_student_xyear
+        description: Defining key for the student, which is consistent across years.
+      - name: k_lea
+        description: Unique key identifying the Local Education Agency (LEA) associated with this disability record.
+      - name: k_school
+        description: Unique key identifying the school associated with this disability record.
+      - name: k_program
+        description: >
+          Unique identifier for the program associated with this disability. Foreign key reference to
+          [dim_program](#!/model/model.edu_wh.dim_program). Null for ed org disability rows.
+      - name: k_student_program
+        description: >
+          Surrogate key for the student's program enrollment. Use this to join to
+          `fct_student_{program_type}_program_association` tables. Null for ed org disability rows.
+      - name: tenant_code
+        description: The tenant associated with the record.
+      - name: school_year
+        description: >
+          School year specified by Spring year.
+          e.g. the 2021-2022 year would be 2022.
+      - name: is_program
+        description: Whether the disability is sourced from a program association (true) or an ed org association (false).
+      - name: program_enroll_begin_date
+        description: The date the student began enrollment in the program associated with this disability. Null for ed org disability rows.
+      - name: program_enroll_end_date
+        description: The date the student's enrollment in the associated program ended, if applicable. Null for ed org disability rows.
+      - name: disability_type
+        description: A disability category that describes the student's impairment (e.g. Autism, Intellectual Disability).
+      - name: disability_source_type
+        description: The source that provided the disability determination (e.g. Medical, Self).
+      - name: disability_diagnosis
+        description: A more specific description of the disability diagnosis, if provided.
+      - name: order_of_disability
+        description: The ordinal ranking when a student has multiple disabilities listed, where 1 indicates the primary disability.