diff --git a/DataQualityExternalModule.php b/DataQualityExternalModule.php index b8d0f32..b8ca015 100644 --- a/DataQualityExternalModule.php +++ b/DataQualityExternalModule.php @@ -15,4 +15,12 @@ public function checkApiToken() { $post = $data->getRequestVars(); } -} \ No newline at end of file + + public function getProcessJsonDownURL() { + return $this->getUrl("exportJsonFile.php", false, false); + } + + public function getProcessJsonUpURL() { + return $this->getUrl("importJsonFile.php", false, false); + } +} diff --git a/README.md b/README.md index 7d53246..5a4f54f 100644 --- a/README.md +++ b/README.md @@ -91,4 +91,9 @@ The data quality import only allows the import of new resolutions. A new status Input for data quality import is identical to the data quality export's output. Duplicate resolutions will not be imported (those with the same ts and username as existing resolutions). -Output for data quality import is a json array of [res_ids]. \ No newline at end of file +Output for data quality import is a json array of [res_ids]. + +## Syncing Data Resolutions between projects +When importing data resolutions using the index.php plugin, you will have the option to "Ignore PID and Usernames?" + +This option should be used only in situations where you are syncing resolutions between projects that have the same data dictionaries and data, but differing users and PIDs. diff --git a/config.json b/config.json index 4f095e8..3a9fe2d 100644 --- a/config.json +++ b/config.json @@ -12,12 +12,23 @@ "institution": "Vanderbilt University Medical Center" } ], - + "permissions": [ ], "no-auth-pages":["import","export"], "project-settings": [ - ] + ], + + "links": { + "project": [ + { + "name": "Data Quality API", + "icon": "arrow_circle_double_135", + "url": "index.php", + "show-header-and-footer": true + } + ] + } } diff --git a/exportJsonFile.php b/exportJsonFile.php new file mode 100644 index 0000000..1dcc08e --- /dev/null +++ b/exportJsonFile.php @@ -0,0 +1,136 @@ + NULL); + +## Read all the rows from the table ino the $statusList variable +while($row = db_fetch_assoc($q)) { + $userId = $row['assigned_user_id']; + unset($row['assigned_user_id']); + + ## Cache User ID to username conversion to reduce DB calls + if(!array_key_exists($userId,$userIdConversion)) { + $userIdConversion[$userId] = User::getUserInfoByUiid($userId)['username']; + } + $row['assigned_username'] = $userIdConversion[$userId]; + $statusList[$row['status_id']] = $row; +} + +if(count($statusList) > 0) { + $sql = "SELECT r.* + FROM redcap_data_quality_resolutions r + WHERE r.status_id IN ('".implode("','",array_keys($statusList))."')"; + + $q = db_query($sql); + + if($e = db_error()) { + throw new Exception("Database error while pulling data quality resolutions"); + } + + while($row = db_fetch_assoc($q)) { + if(!array_key_exists("resolutions",$statusList[$row['status_id']])) { + $statusList[$row['status_id']]["resolutions"] = array(); + } + $userId = $row['user_id']; + unset($row['user_id']); + + ## Cache User ID to username conversion to reduce DB calls + if(!array_key_exists($userId,$userIdConversion)) { + $userIdConversion[$userId] = User::getUserInfoByUiid($userId)['username']; + } + $row['username'] = $userIdConversion[$userId]; + $statusList[$row['status_id']]["resolutions"][$row["res_id"]] = $row; + } +} + +// $uid = User::getUIIDByUsername($post['user']); + +// $retData = array( +// 'uid' => $uid, +// 'user' => $post['user'], +// 'sql' => $sql +// ); + +// $content = json_encode($sql1); +$content = json_encode($statusList); + +# Export to a JSON file +$datestamp = new DateTime(); +$filename=$project_id."-data_quality-".$datestamp->format("Y-m-d").".json"; + +// $fp = fopen($filename, 'w'); +// fwrite($fp, $content); +// fclose($fp); + +header("Content-type: application/json"); +header("Content-Disposition: attachment; filename=".$filename); +header("Pragma: no-cache"); +header("Expires: 0"); + +echo($content); diff --git a/importJsonFile.php b/importJsonFile.php new file mode 100644 index 0000000..e36156c --- /dev/null +++ b/importJsonFile.php @@ -0,0 +1,516 @@ +setFormLevelPrivileges(); + +$Proj = new Project(PROJECT_ID); +$errors = array(); +$notes = array(); + +// Prevent data imports for projects in inactive or archived status +if ($Proj->project['status'] > 1) { + if ($Proj->project['status'] == '2') { + $statusLabel = "Inactive"; + } elseif ($Proj->project['status'] == '3') { + $statusLabel = "Archived"; + } else { + $statusLabel = "[unknown]"; + } + array_push($errors, "Data may not be imported because the project is in $statusLabel status."); +} + +$insertValues = []; +$resolutionInsertIds = []; +$userIdConversion = array(NULL => NULL); + +$projectSettings = $Proj->project; +$dataQualityProcess = false; +$dataCommentProcess = false; +$editComments = false; + +## Check if project has data resolution workflow turned on and ability to delete data resolution comments +if($projectSettings['data_resolution_enabled'] == 2) { + $dataQualityProcess = true; +} +else if($projectSettings['data_resolution_enabled'] == 1) { + $dataCommentProcess = true; +} +else { + array_push($errors, "Data may not be imported because the project does not have data quality enabled."); +} + +if($projectSettings['field_comment_edit_delete'] == 1) { + $editComments = true; +} + +$single_event = false; +if(sizeof($Proj->events)==1 && $projectSettings['repeatforms']==0){ + $single_event = true; +} + +// Import the json file +if (isset($_FILES['import_file']) && $_FILES['import_file']['error'] === UPLOAD_ERR_OK) { + $importData = json_decode(file_get_contents($_FILES['import_file']['tmp_name']),true); +}else{ + array_push($errors, "File failed to upload"); +} + +if(is_null($importData)){ + array_push($errors, "Failed to decode json. Please ensure that the file selected was json..."); + $content = NULL; +}else{ + foreach($importData as $dataRow) { + $statusId = $dataRow['status_id']; + $record = $dataRow['record']; + $projectId = $dataRow['project_id']; + $eventId = $dataRow['event_id']; + $fieldName = $dataRow['field_name']; + $instance = $dataRow['instance']; + $repeatInstrument = $dataRow['repeat_instrument']; + $assignedUsername = $dataRow['assigned_username']; + $newStatusId = false; + + ## Verify field name and event ID exist on this project. Skip this row if it doesn't + if(!array_key_exists($fieldName,$Proj->metadata)) { + continue; + } + + $eventFound = false; + foreach($Proj->events as $armDetails) { + foreach($armDetails["events"] as $tempEventId => $eventDetails) { + // check for non-longitudinal project + if (count($armDetails['events']) == 1 && $single_event){ + $eventId = $tempEventId; + $eventFound = true; + break 2; + }else if($eventId == $tempEventId) { + $eventFound = true; + break 2; + } + } + } + + if(!$eventFound) { + array_push($notes, "Event not found."); + continue; + } + + ## Skip if record/project/event/field is blank + if($record == "" || $projectId == "" || $eventId == "" || $fieldName == "") continue; + + ## Skip data where the projectId doesn't match the token's projectId + if($projectId != PROJECT_ID && is_null($ignore)){ + continue; + }else{ + ## Allow imports from other projects--bypassing project ID + $ignored_project_id = $projectId; + $projectId = PROJECT_ID; + } + + ## Confirm if this status is already in the database + $sql = "SELECT s.status_id, s.instance, s.repeat_instrument, s.record, s.project_id, s.event_id, s.field_name + FROM redcap_data_quality_status s + WHERE ".(($statusId == "" || !is_numeric($statusId)) ? "" : "s.status_id = '".db_escape($statusId)."'")." + OR (s.record = '".db_escape($record)."' + AND s.project_id = '".db_escape($projectId)."' + AND s.event_id = '".db_escape($eventId)."' + AND s.field_name = '".db_escape($fieldName)."'". + ($repeatInstrument != NULL ? " AND s.repeat_instrument = '".db_escape($repeatInstrument)."'" : + ($instance == NULL ? " AND s.instance is NULL" : " AND s.instance = '".db_escape($instance)."'")).")"; + + $q = db_query($sql); + ## Cache User ID to username conversion to reduce DB calls + if(!array_key_exists($assignedUsername,$userIdConversion)) { + if(User::getUIIDByUsername($assignedUsername) || is_null($ignore)){ + $userIdConversion[$assignedUsername] = User::getUIIDByUsername($assignedUsername); + }else{ + // Ignore the user id imported and instead overwrite with current user + $ignored_username = $username; + $userIdConversion[$assignedUsername] = User::getUIIDByUsername(USERID); + } + } + + $existingStatus = ""; + while($row = db_fetch_assoc($q)) { + ## Found a matching record/project/event/field/instance with a different status + if($row['status_id'] != $statusId) { + array_push($notes, "Found a matching record/project/event/field/instance with a different status"); + $existingStatus = $row['status_id']; + $newStatusId = true; + } + ## Found a row for that statusId, verify if record/project/event/field/instance matches + else if($row['record'] == $record && ($row['project_id'] == $projectId || !is_null($ignore)) + && $row['event_id'] == $eventId && $row['field_name'] == $fieldName){ + if($instance == "" || $row['instance'] == $instance) { + array_push($notes, "Found a row for this statusId, verified that record/project/event/field/instance matches"); + $existingStatus = $row['status_id']; + $newStatusId = false; + } + } + } + + ## Add new status row + if($existingStatus == "" || is_null($existingStatus)) { + array_push($errors,$record_esc); + $sql = "INSERT INTO redcap_data_quality_status + (non_rule,project_id,record,event_id,field_name,instance,assigned_user_id) + VALUES (1,".db_escape($projectId).",'".db_escape($record)."',".db_escape($eventId).",'". + db_escape($fieldName)."',".checkNull($instance).",".checkNull($userIdConversion[$assignedUsername]).")"; + $q = db_query($sql); + if($e = db_error()) { + array_push($errors,"sql: ".$sql); + array_push($errors,"Insert Status error: ".$e); + } + $existingStatus = db_insert_id(); + $newStatusId = true; + } + + ## Do date conversion on resolutions so they can be quickly sorted + foreach($dataRow['resolutions'] as $resKey => $resolutionRow) { + $dataRow['resolutions'][$resKey]['tsInSeconds'] = strtotime($resolutionRow['ts']); + } + + ## Sort resolutions by timestamp and get last timestamp in DB + usort($dataRow['resolutions'],function($a,$b) { + if($a['tsInSeconds'] > $b['tsInSeconds']) { + return 1; + } + else if($a['tsInSeconds'] == $b['tsInSeconds']) { + return 0; + } + return -1; + }); + + $sql = "SELECT r.ts,r.user_id + FROM redcap_data_quality_resolutions r + WHERE r.status_id = '".$existingStatus."' + ORDER BY r.ts DESC"; + + $q = db_query($sql); + $existingResolutions = array(); + while($row = db_fetch_assoc($q)) { + $existingResolutions[$row['ts'].$row['user_id']] = 1; + } + + $dbTs = db_result($q,0,"ts"); + $lastTs = ""; + + foreach($dataRow['resolutions'] as $resolutionRow) { + + // Determine the status to set + if (in_array($resolutionRow['current_query_status'], array('OPEN','CLOSED','VERIFIED','DEVERIFIED'))) { + $resStatus = $resolutionRow['current_query_status']; + } elseif ((isset($resolutionRow['response_requested']) && $resolutionRow['response_requested']) + || ($resolutionRow['response'] && $resolutionRow['response'])) { + $resStatus = 'OPEN'; + } else { + $resStatus = ''; + } + + // Make sure response is in enum list + if(in_array($resolutionRow['response'],array('DATA_MISSING','TYPOGRAPHICAL_ERROR','CONFIRMED_CORRECT','WRONG_SOURCE','OTHER'))) { + $response = $resolutionRow['response']; + } + else { + $response = ''; + } + + $resolutionId = $resolutionRow['res_id']; + $resStatusId = $resolutionRow['status_id']; + $username = $resolutionRow['username']; + array_push($notes,"Old res_id=".$resolutionId."; res_status_id=".$resStatusId); + + ## Skip resolution rows that don't have matching status IDs unless this is a new status + if(!$newStatusId && $resStatusId != "" && $resStatusId != $existingStatus){ + array_push($notes, "Skip resolution rows that don't have matching status IDs unless this is a new status"); + continue; + } + + ## Cache User ID to username conversion to reduce DB calls + if(!array_key_exists($username,$userIdConversion)) { + if(User::getUIIDByUsername($assignedUsername) || is_null($ignore)){ + $userIdConversion[$assignedUsername] = User::getUIIDByUsername($assignedUsername); + }else{ + // Ignore the user id imported and instead overwrite with current user + $ignored_username = $username; + $userIdConversion[$assignedUsername] = User::getUIIDByUsername(USERID); + } + } + + ## Skip resolution rows that already have a resolution from the same user for the same timestamp + ## This is to prevent duplicate response rows + if(array_key_exists($resolutionRow['ts'].$userIdConversion[$username],$existingResolutions)) { + array_push($notes,"Skipping resolution row--already has a resolution from the same user for the same timestamp"); + continue; + } + + $lastTs = $resolutionRow['ts']; + + # append the userid to the comments if it will be ignored + $append_comment = $resolutionRow['comment']; + if(!is_null($ignore)){ + $add_info = " (DRW Import PID:".$ignored_project_id.", "; + $add_info .= "RESID:".$resolutionId.", "; + $add_info .= "UID:".$ignored_username.")"; + array_push($notes, $add_info); + } + $append_comment .= $add_info; + + $insertValues[] = "(".checkNull($existingStatus).",".checkNull($resolutionRow['ts']).",". + checkNull($userIdConversion[$username]).",".(in_array($resolutionRow['response_requested'],[0,1]) ? $resolutionRow['response_requested'] : 0).",". + checkNull($response).",".checkNull($append_comment).",". + checkNull($resStatus).")"; + + $existingResolutions[$resolutionRow['ts'].$userIdConversion[$username]] = 1; + + ## Do inserts in groups of 10 to reduce DB calls + if(count($insertValues) >= 10) { + array_push($notes, "Inserting resolutions in batches of 10."); + $sql = "INSERT INTO redcap_data_quality_resolutions + (status_id,ts,user_id,response_requested,response,comment,current_query_status) + VALUES ".implode(",",$insertValues); + db_query($sql); + + if($e = db_error()) { + error_log("Data Quality Import: ".$e); + } + else { + $nextId = db_insert_id(); + for($i = 0; $i < 10; $i++) { + $resolutionInsertIds[] = $nextId + $i; + } + } + $insertValues = []; + } + } + + ## If last timestamp is after last timestamp in DB, then update main query_status + if($existingStatus && $lastTs != "" && strtotime($lastTs) > strtotime($dbTs)) { + $sql = "UPDATE redcap_data_quality_status + SET query_status = ".checkNull($resStatus)." + WHERE status_id = ".$existingStatus; + + db_query($sql); + } + array_push($notes, "________________________"); + } # end foreach json row + + if(count($insertValues) > 0) { + ## Do last inserts pending in $insertValues + array_push($notes, "Inserting final set of resolutions."); + $sql = "INSERT INTO redcap_data_quality_resolutions + (status_id,ts,user_id,response_requested,response,comment,current_query_status) + VALUES ".implode(",",$insertValues); + db_query($sql); + + if($e = db_error()) { + error_log("Data Quality Import: ".$e); + array_push($errors,$e); + } + else { + $nextId = db_insert_id(); + for($i = 0; $i < count($insertValues); $i++) { + $resolutionInsertIds[] = $nextId + $i; + } + } + $insertValues = []; + } + + $content = json_encode($resolutionInsertIds); + + if(!is_null($file_repo)){ + # Save the file to the repository + array_push($notes,"Attempting to save to file repository..."); + $info_to_save = "MODIFIED CONTENT = ".implode(";",$content)."\n\n"."NOTES = ".implode(";",$notes)."\n\n"."ERRORS = ".implode(";",$errors)."\n\n"."JSON = ".json_encode($importData); + $saved = saveToFileRepository("Data Quality API Log", $info_to_save, "txt"); + } +} + + +if (!empty($errors)) +{ + require_once APP_PATH_DOCROOT . 'ProjectGeneral/header.php'; + print "
$error
"; + } + print "RES IDS MODIFIED:$content
"; + print "-------------LOG:-------------
"; + foreach($notes as $note){ + print ""; + print_r($note); + print "
"; + } + print "SUCCESS! RES IDS MODIFIED:$content
"; + print "-------------LOG:-------------
"; + foreach($notes as $note){ + print ""; + print_r($note); + print "
"; + } + print "Process completed, but no new resolutions were added. Is that what you expected?
"; + } + if (!empty($saved)){ + print "
+
+
+ +