Fix Pandas compatibility, DCID collisions, and validation config for NCES_SchoolDistrict and NCES_PublicSchool

scripts/us_nces/common/prop_conf.py

@@ -18,6 +18,8 @@
 While preprocessing files column names are changed to SV names as used in
 DC import
 """
+import pandas as pd
+
 # TMCF template for Demographics data. It changes based on import name.
 TMCF_TEMPLATE = (
     "Node: E:us_nces_demographics_{import_name}->E0\n"
@@ -123,17 +125,25 @@
     "School Administrative Support Staff": "Faculty",
     "Student Support Services Staff": "Faculty",
     "School Psychologist": "Faculty",
-    "Other Support Services Staff": "Faculty"
+    "Other Support Services Staff": "Faculty",
+    "Total Staff": "Faculty"
 }
 # One specific column comes under school grade property.
 _SCHOOL_GRADE_PROP = {"Ungraded Students": "NCESUngradedClasses"}
 # melting the columns based on sv_name column.
 MELT_VAR_COL = "sv_name"
 
+
+def _PV_FORMAT(pv):
+    """Formats property-value pairs for MCF nodes; modified based on column."""
+    t = tuple(pv)
+    val = str(t[1]).strip() if not pd.isna(t[1]) else ""
+    if not val or val in ('None', 'nan', '<NA>'):
+        return ""
+    return f'"{t[0]}": "dcs:{val}"'
+
+
 # pylint:disable=unnecessary-lambda-assignment
-# Creating property pattern and the pattern is modified if required based on column.
-_PV_FORMAT = lambda prop_val: f'"{prop_val[0]}": "dcs:{prop_val[1]}"' \
-                        if 'None' not in prop_val[1] else ""
 _UPDATE_MEASUREMENT_DENO = lambda prop: _DENOMINATOR_PROP.get(prop, prop)
 _UPDATE_POPULATION_TYPE = lambda prop: _POPULATION_PROP.get(prop, "Student")
 _UPDATE_GRADE_LEVEL = lambda prop: _SCHOOL_GRADE_PROP.get(prop, prop)
@@ -167,7 +177,7 @@
                  r")")
 
 _SCHOOL_GRADE_PATTERN = (r"("
-                         r"Grade \d{,2}"
+                         r"Grade \d{1,2}"
                          r"|"
                          r"Prekindergarten and Kindergarten"
                          r"|"
@@ -236,6 +246,8 @@
                             r"School Psychologist"
                             r"|"
                             r"Other Support Services Staff"
+                            r"|"
+                            r"Total Staff"
                             r")")
 
 _GENDER_PATTERN = (r"("

scripts/us_nces/common/replacement_functions.py

@@ -372,7 +372,7 @@
     "Phone Number": "PhoneNumber"
 }
 
-_GENDER = {"female": "Female", "male": "Male"}
+_GENDER = {r"\bfemale\b": "Female", r"\bmale\b": "Male"}
 
 _LOCALE = {
     '13-City: Small': "NCES_CitySmall",
@@ -501,6 +501,7 @@ def replace_values(data_df: pd.DataFrame,
         "Agency_Name": _NAN,
         "School_Level_17": _SCHOOL_LEVEL,
         "School_Level_16": _SCHOOL_LEVEL,
+        "School_Level": _SCHOOL_LEVEL,
         "State_Agency_ID": _NAN,
         "State_School_ID": _NAN,
         "State_Name": _STATE_NAME