From 01a551f92fe3a073ecd97e048bf29e4ede7cd5a5 Mon Sep 17 00:00:00 2001 From: devincowan Date: Mon, 5 Oct 2020 11:24:18 -0600 Subject: [PATCH 1/8] add JSON file export start work on JSON file import --- DataQualityExternalModule.php | 307 ++++++++++++++++++++++++++++++++- config.json | 15 +- exportJsonFile.php | 136 +++++++++++++++ importJsonFile.php | 316 ++++++++++++++++++++++++++++++++++ index.php | 39 +++++ 5 files changed, 810 insertions(+), 3 deletions(-) create mode 100644 exportJsonFile.php create mode 100644 importJsonFile.php create mode 100644 index.php diff --git a/DataQualityExternalModule.php b/DataQualityExternalModule.php index b8d0f32..61d81ba 100644 --- a/DataQualityExternalModule.php +++ b/DataQualityExternalModule.php @@ -15,4 +15,309 @@ public function checkApiToken() { $post = $data->getRequestVars(); } -} \ No newline at end of file + + public function getProcessJsonDownURL() { + return $this->getUrl("exportJsonFile.php", false, false); + } + + public function getProcessJsonUpURL() { + return $this->getUrl("importJsonFile.php", false, false); + } + + public function import_json_file() { + global $format, $returnFormat, $post; + + // Get user's user rights + $user_rights = UserRights::getPrivileges(PROJECT_ID, USERID); + $user_rights = $user_rights[PROJECT_ID][strtolower(USERID)]; + $ur = new UserRights(); + $ur->setFormLevelPrivileges(); + + $Proj = new Project(PROJECT_ID); + $errors = array(); + + // Prevent data imports for projects in inactive or archived status + if ($Proj->project['status'] > 1) { + if ($Proj->project['status'] == '2') { + $statusLabel = "Inactive"; + } elseif ($Proj->project['status'] == '3') { + $statusLabel = "Archived"; + } else { + $statusLabel = "[unknown]"; + } + array_push($errors, "Data may not be imported because the project is in $statusLabel status."); + } + + $insertValues = []; + $resolutionInsertIds = []; + $userIdConversion = array(NULL => NULL); + + $projectSettings = $Proj->project; + $dataQualityProcess = false; + $dataCommentProcess = false; + $editComments = false; + + ## Check if project has data resolution workflow turned on and ability to delete data resolution comments + if($projectSettings['data_resolution_enabled'] == 2) { + $dataQualityProcess = true; + } + else if($projectSettings['data_resolution_enabled'] == 1) { + $dataCommentProcess = true; + } + else { + array_push($errors, "Data may not be imported because the project does not have data quality enabled.")); + } + + if($projectSettings['field_comment_edit_delete'] == 1) { + $editComments = true; + } + + $single_event = false; + if(sizeof($Proj->events)==1 && $projectSettings['repeatforms']==0){ + $single_event = true; + } + + ## TODO Convert flat/csv data into nested array format + $importData = json_decode($post['import_file'],true); + + foreach($importData as $dataRow) { + $statusId = $dataRow['status_id']; + $record = $dataRow['record']; + $projectId = $dataRow['project_id']; + $eventId = $dataRow['event_id']; + $fieldName = $dataRow['field_name']; + $instance = $dataRow['instance']; + $repeatInstrument = $dataRow['repeat_instrument']; + $assignedUsername = $dataRow['assigned_username']; + $newStatusId = false; + + ## Verify field name and event ID exist on this project. Skip this row if it doesn't + if(!array_key_exists($fieldName,$Proj->metadata)) { + continue; + } + + $eventFound = false; + foreach($Proj->events as $armDetails) { + foreach($armDetails["events"] as $tempEventId => $eventDetails) { + // check for non-longitudinal project + if (count($armDetails['events']) == 1 && $single_event){ + $eventId = $tempEventId; + $eventFound = true; + break 2; + }else if($eventId == $tempEventId) { + $eventFound = true; + break 2; + } + } + } + + if(!$eventFound) { + continue; + } + + ## Skip if record/project/event/field is blank + if($record == "" || $projectId == "" || $eventId == "" || $fieldName == "") continue; + + ## Skip data where the projectId doesn't match the token's projectId + if($projectId != PROJECT_ID) continue; + + + ## Confirm if this status is already in the database + $sql = "SELECT s.status_id, s.instance, s.repeat_instrument, s.record, s.project_id, s.event_id, s.field_name + FROM redcap_data_quality_status s + WHERE ".(($statusId == "" || !is_numeric($statusId)) ? "" : "s.status_id = '".db_escape($statusId)."'")." + OR (s.record = '".db_escape($record)."' + AND s.project_id = '".db_escape($projectId)."' + AND s.event_id = '".db_escape($eventId)."' + AND s.field_name = '".db_escape($fieldName)."'". + ($repeatInstrument != NULL ? " AND s.repeat_instrument = '".db_escape($repeatInstrument)."'" : + ($instance == NULL ? " AND s.instance is NULL" : " AND s.instance = '".db_escape($instance)."'")).")"; + + $q = db_query($sql); + + ## Cache User ID to username conversion to reduce DB calls + if(!array_key_exists($assignedUsername,$userIdConversion)) { + $userIdConversion[$assignedUsername] = User::getUIIDByUsername($assignedUsername); + } + + $existingStatus = ""; + while($row = db_fetch_assoc($q)) { + ## Found a matching record/project/event/field/instance with a different status + if($row['status_id'] != $statusId) { + $existingStatus = $row['status_id']; + $newStatusId = true; + } + ## Found a row for that statusId, verify if record/project/event/field/instance matches + else if($row['record'] == $record && $row['project_id'] == $projectId + && $row['event_id'] == $eventId && $row['field_name'] == $fieldName){ + if($instance == "" || $row['instance'] == $instance) { + $existingStatus = $row['status_id']; + $newStatusId = false; + } + } + } + + ## Add new status row + if($existingStatus == "") { + $sql = "INSERT INTO redcap_data_quality_status + (non_rule,project_id,record,event_id,field_name,instance,assigned_user_id) + VALUES (1,".db_escape($projectId).",".db_escape($record).",".db_escape($eventId).",'". + db_escape($fieldName)."',".checkNull($instance).",".checkNull($userIdConversion[$assignedUsername]).")"; + + db_query($sql); + $existingStatus = db_insert_id(); + $newStatusId = true; + } + + ## Do date conversion on resolutions so they can be quickly sorted + foreach($dataRow['resolutions'] as $resKey => $resolutionRow) { + $dataRow['resolutions'][$resKey]['tsInSeconds'] = strtotime($resolutionRow['ts']); + } + + ## Sort resolutions by timestamp and get last timestamp in DB + usort($dataRow['resolutions'],function($a,$b) { + if($a['tsInSeconds'] > $b['tsInSeconds']) { + return 1; + } + else if($a['tsInSeconds'] == $b['tsInSeconds']) { + return 0; + } + return -1; + }); + + $sql = "SELECT r.ts,r.user_id + FROM redcap_data_quality_resolutions r + WHERE r.status_id = '".$existingStatus."' + ORDER BY r.ts DESC"; + + $q = db_query($sql); + $existingResolutions = array(); + while($row = db_fetch_assoc($q)) { + $existingResolutions[$row['ts'].$row['user_id']] = 1; + } + + $dbTs = db_result($q,0,"ts"); + $lastTs = ""; + + foreach($dataRow['resolutions'] as $resolutionRow) { + if($editComments) { + # TODO Allow update of data resolution status/comments + //DataQuality::editFieldComment(); + } + + // Determine the status to set + if (in_array($resolutionRow['current_query_status'], array('OPEN','CLOSED','VERIFIED','DEVERIFIED'))) { + $resStatus = $resolutionRow['current_query_status']; + } elseif ((isset($resolutionRow['response_requested']) && $resolutionRow['response_requested']) + || ($resolutionRow['response'] && $resolutionRow['response'])) { + $resStatus = 'OPEN'; + } else { + $resStatus = ''; + } + + // Make sure response is in enum list + if(in_array($resolutionRow['response'],array('DATA_MISSING','TYPOGRAPHICAL_ERROR','CONFIRMED_CORRECT','WRONG_SOURCE','OTHER'))) { + $response = $resolutionRow['response']; + } + else { + $response = ''; + } + + $resolutionId = $resolutionRow['res_id']; + $resStatusId = $resolutionRow['status_id']; + $username = $resolutionRow['username']; + + ## Skip resolution rows that don't have matching status IDs unless this is a new status + if(!$newStatusId && $resStatusId != "" && $resStatusId != $existingStatus) continue; + + ## Cache User ID to username conversion to reduce DB calls + if(!array_key_exists($username,$userIdConversion)) { + $userIdConversion[$username] = User::getUIIDByUsername($username); + } + + ## Skip resolution rows that already have a resolution from the same user for the same timestamp + ## This is to prevent duplicate response rows + if(array_key_exists($resolutionRow['ts'].$userIdConversion[$username],$existingResolutions)) { + continue; + } + + $lastTs = $resolutionRow['ts']; + + $insertValues[] = "(".checkNull($existingStatus).",".checkNull($resolutionRow['ts']).",". + checkNull($userIdConversion[$username]).",".(in_array($resolutionRow['response_requested'],[0,1]) ? $resolutionRow['response_requested'] : 0).",". + checkNull($response).",".checkNull($resolutionRow['comment']).",". + checkNull($resStatus).")"; + + $existingResolutions[$resolutionRow['ts'].$userIdConversion[$username]] = 1; + + ## Do inserts in groups of 10 to reduce DB calls + if(count($insertValues) >= 10) { + $sql = "INSERT INTO redcap_data_quality_resolutions + (status_id,ts,user_id,response_requested,response,comment,current_query_status) + VALUES ".implode(",",$insertValues); + db_query($sql); + + if($e = db_error()) { + error_log("Data Quality Import: ".$e); + } + else { + $nextId = db_insert_id(); + for($i = 0; $i < 10; $i++) { + $resolutionInsertIds[] = $nextId + $i; + } + } + $insertValues = []; + } + } + + ## If last timestamp is after last timestamp in DB, then update main query_status + if($existingStatus && $lastTs != "" && strtotime($lastTs) > strtotime($dbTs)) { + $sql = "UPDATE redcap_data_quality_status + SET query_status = ".checkNull($resStatus)." + WHERE status_id = ".$existingStatus; + + db_query($sql); + } + } + + if(count($insertValues) > 0) { + ## Do last inserts pending in $insertValues + $sql = "INSERT INTO redcap_data_quality_resolutions + (status_id,ts,user_id,response_requested,response,comment,current_query_status) + VALUES ".implode(",",$insertValues); + db_query($sql); + + if($e = db_error()) { + error_log("Data Quality Import: ".$e); + } + else { + $nextId = db_insert_id(); + for($i = 0; $i < count($insertValues); $i++) { + $resolutionInsertIds[] = $nextId + $i; + } + } + $insertValues = []; + } + + ## TODO Send other types of responses based on request + $content = json_encode($resolutionInsertIds); + return ($errors); + + function csv($data,$headers) { + foreach($data as $dataRow) { + foreach($headers as $column => $label) { + if($label == "record") { + ## If record is blank, this must be a data resolution row + if($dataRow[$column] == "") { + + } + ## Else, this must be a status row + else { + + } + } + } + } + } + } +} diff --git a/config.json b/config.json index 4f095e8..98eb482 100644 --- a/config.json +++ b/config.json @@ -12,12 +12,23 @@ "institution": "Vanderbilt University Medical Center" } ], - + "permissions": [ ], "no-auth-pages":["import","export"], "project-settings": [ - ] + ], + + "links": { + "project": [ + { + "name": "Export/Import DRW JSON", + "icon": "arrow_circle_double_135", + "url": "index.php", + "show-header-and-footer": true + } + ] + } } diff --git a/exportJsonFile.php b/exportJsonFile.php new file mode 100644 index 0000000..1dcc08e --- /dev/null +++ b/exportJsonFile.php @@ -0,0 +1,136 @@ + NULL); + +## Read all the rows from the table ino the $statusList variable +while($row = db_fetch_assoc($q)) { + $userId = $row['assigned_user_id']; + unset($row['assigned_user_id']); + + ## Cache User ID to username conversion to reduce DB calls + if(!array_key_exists($userId,$userIdConversion)) { + $userIdConversion[$userId] = User::getUserInfoByUiid($userId)['username']; + } + $row['assigned_username'] = $userIdConversion[$userId]; + $statusList[$row['status_id']] = $row; +} + +if(count($statusList) > 0) { + $sql = "SELECT r.* + FROM redcap_data_quality_resolutions r + WHERE r.status_id IN ('".implode("','",array_keys($statusList))."')"; + + $q = db_query($sql); + + if($e = db_error()) { + throw new Exception("Database error while pulling data quality resolutions"); + } + + while($row = db_fetch_assoc($q)) { + if(!array_key_exists("resolutions",$statusList[$row['status_id']])) { + $statusList[$row['status_id']]["resolutions"] = array(); + } + $userId = $row['user_id']; + unset($row['user_id']); + + ## Cache User ID to username conversion to reduce DB calls + if(!array_key_exists($userId,$userIdConversion)) { + $userIdConversion[$userId] = User::getUserInfoByUiid($userId)['username']; + } + $row['username'] = $userIdConversion[$userId]; + $statusList[$row['status_id']]["resolutions"][$row["res_id"]] = $row; + } +} + +// $uid = User::getUIIDByUsername($post['user']); + +// $retData = array( +// 'uid' => $uid, +// 'user' => $post['user'], +// 'sql' => $sql +// ); + +// $content = json_encode($sql1); +$content = json_encode($statusList); + +# Export to a JSON file +$datestamp = new DateTime(); +$filename=$project_id."-data_quality-".$datestamp->format("Y-m-d").".json"; + +// $fp = fopen($filename, 'w'); +// fwrite($fp, $content); +// fclose($fp); + +header("Content-type: application/json"); +header("Content-Disposition: attachment; filename=".$filename); +header("Pragma: no-cache"); +header("Expires: 0"); + +echo($content); diff --git a/importJsonFile.php b/importJsonFile.php new file mode 100644 index 0000000..01366c1 --- /dev/null +++ b/importJsonFile.php @@ -0,0 +1,316 @@ +setFormLevelPrivileges(); + +$Proj = new Project(PROJECT_ID); +$errors = array(); + +// Prevent data imports for projects in inactive or archived status +if ($Proj->project['status'] > 1) { + if ($Proj->project['status'] == '2') { + $statusLabel = "Inactive"; + } elseif ($Proj->project['status'] == '3') { + $statusLabel = "Archived"; + } else { + $statusLabel = "[unknown]"; + } + array_push($errors, "Data may not be imported because the project is in $statusLabel status."); +} + +$insertValues = []; +$resolutionInsertIds = []; +$userIdConversion = array(NULL => NULL); + +$projectSettings = $Proj->project; +$dataQualityProcess = false; +$dataCommentProcess = false; +$editComments = false; + +## Check if project has data resolution workflow turned on and ability to delete data resolution comments +if($projectSettings['data_resolution_enabled'] == 2) { + $dataQualityProcess = true; +} +else if($projectSettings['data_resolution_enabled'] == 1) { + $dataCommentProcess = true; +} +else { + array_push($errors, "Data may not be imported because the project does not have data quality enabled.")); +} + +if($projectSettings['field_comment_edit_delete'] == 1) { + $editComments = true; +} + +$single_event = false; +if(sizeof($Proj->events)==1 && $projectSettings['repeatforms']==0){ + $single_event = true; +} + +## TODO Convert flat/csv data into nested array format +$importData = json_decode($post['import_file'],true); + +foreach($importData as $dataRow) { + $statusId = $dataRow['status_id']; + $record = $dataRow['record']; + $projectId = $dataRow['project_id']; + $eventId = $dataRow['event_id']; + $fieldName = $dataRow['field_name']; + $instance = $dataRow['instance']; + $repeatInstrument = $dataRow['repeat_instrument']; + $assignedUsername = $dataRow['assigned_username']; + $newStatusId = false; + + ## Verify field name and event ID exist on this project. Skip this row if it doesn't + if(!array_key_exists($fieldName,$Proj->metadata)) { + continue; + } + + $eventFound = false; + foreach($Proj->events as $armDetails) { + foreach($armDetails["events"] as $tempEventId => $eventDetails) { + // check for non-longitudinal project + if (count($armDetails['events']) == 1 && $single_event){ + $eventId = $tempEventId; + $eventFound = true; + break 2; + }else if($eventId == $tempEventId) { + $eventFound = true; + break 2; + } + } + } + + if(!$eventFound) { + continue; + } + + ## Skip if record/project/event/field is blank + if($record == "" || $projectId == "" || $eventId == "" || $fieldName == "") continue; + + ## Skip data where the projectId doesn't match the token's projectId + if($projectId != PROJECT_ID) continue; + + + ## Confirm if this status is already in the database + $sql = "SELECT s.status_id, s.instance, s.repeat_instrument, s.record, s.project_id, s.event_id, s.field_name + FROM redcap_data_quality_status s + WHERE ".(($statusId == "" || !is_numeric($statusId)) ? "" : "s.status_id = '".db_escape($statusId)."'")." + OR (s.record = '".db_escape($record)."' + AND s.project_id = '".db_escape($projectId)."' + AND s.event_id = '".db_escape($eventId)."' + AND s.field_name = '".db_escape($fieldName)."'". + ($repeatInstrument != NULL ? " AND s.repeat_instrument = '".db_escape($repeatInstrument)."'" : + ($instance == NULL ? " AND s.instance is NULL" : " AND s.instance = '".db_escape($instance)."'")).")"; + + $q = db_query($sql); + + ## Cache User ID to username conversion to reduce DB calls + if(!array_key_exists($assignedUsername,$userIdConversion)) { + $userIdConversion[$assignedUsername] = User::getUIIDByUsername($assignedUsername); + } + + $existingStatus = ""; + while($row = db_fetch_assoc($q)) { + ## Found a matching record/project/event/field/instance with a different status + if($row['status_id'] != $statusId) { + $existingStatus = $row['status_id']; + $newStatusId = true; + } + ## Found a row for that statusId, verify if record/project/event/field/instance matches + else if($row['record'] == $record && $row['project_id'] == $projectId + && $row['event_id'] == $eventId && $row['field_name'] == $fieldName){ + if($instance == "" || $row['instance'] == $instance) { + $existingStatus = $row['status_id']; + $newStatusId = false; + } + } + } + + ## Add new status row + if($existingStatus == "") { + $sql = "INSERT INTO redcap_data_quality_status + (non_rule,project_id,record,event_id,field_name,instance,assigned_user_id) + VALUES (1,".db_escape($projectId).",".db_escape($record).",".db_escape($eventId).",'". + db_escape($fieldName)."',".checkNull($instance).",".checkNull($userIdConversion[$assignedUsername]).")"; + + db_query($sql); + $existingStatus = db_insert_id(); + $newStatusId = true; + } + + ## Do date conversion on resolutions so they can be quickly sorted + foreach($dataRow['resolutions'] as $resKey => $resolutionRow) { + $dataRow['resolutions'][$resKey]['tsInSeconds'] = strtotime($resolutionRow['ts']); + } + + ## Sort resolutions by timestamp and get last timestamp in DB + usort($dataRow['resolutions'],function($a,$b) { + if($a['tsInSeconds'] > $b['tsInSeconds']) { + return 1; + } + else if($a['tsInSeconds'] == $b['tsInSeconds']) { + return 0; + } + return -1; + }); + + $sql = "SELECT r.ts,r.user_id + FROM redcap_data_quality_resolutions r + WHERE r.status_id = '".$existingStatus."' + ORDER BY r.ts DESC"; + + $q = db_query($sql); + $existingResolutions = array(); + while($row = db_fetch_assoc($q)) { + $existingResolutions[$row['ts'].$row['user_id']] = 1; + } + + $dbTs = db_result($q,0,"ts"); + $lastTs = ""; + + foreach($dataRow['resolutions'] as $resolutionRow) { + if($editComments) { + # TODO Allow update of data resolution status/comments + //DataQuality::editFieldComment(); + } + + // Determine the status to set + if (in_array($resolutionRow['current_query_status'], array('OPEN','CLOSED','VERIFIED','DEVERIFIED'))) { + $resStatus = $resolutionRow['current_query_status']; + } elseif ((isset($resolutionRow['response_requested']) && $resolutionRow['response_requested']) + || ($resolutionRow['response'] && $resolutionRow['response'])) { + $resStatus = 'OPEN'; + } else { + $resStatus = ''; + } + + // Make sure response is in enum list + if(in_array($resolutionRow['response'],array('DATA_MISSING','TYPOGRAPHICAL_ERROR','CONFIRMED_CORRECT','WRONG_SOURCE','OTHER'))) { + $response = $resolutionRow['response']; + } + else { + $response = ''; + } + + $resolutionId = $resolutionRow['res_id']; + $resStatusId = $resolutionRow['status_id']; + $username = $resolutionRow['username']; + + ## Skip resolution rows that don't have matching status IDs unless this is a new status + if(!$newStatusId && $resStatusId != "" && $resStatusId != $existingStatus) continue; + + ## Cache User ID to username conversion to reduce DB calls + if(!array_key_exists($username,$userIdConversion)) { + $userIdConversion[$username] = User::getUIIDByUsername($username); + } + + ## Skip resolution rows that already have a resolution from the same user for the same timestamp + ## This is to prevent duplicate response rows + if(array_key_exists($resolutionRow['ts'].$userIdConversion[$username],$existingResolutions)) { + continue; + } + + $lastTs = $resolutionRow['ts']; + + $insertValues[] = "(".checkNull($existingStatus).",".checkNull($resolutionRow['ts']).",". + checkNull($userIdConversion[$username]).",".(in_array($resolutionRow['response_requested'],[0,1]) ? $resolutionRow['response_requested'] : 0).",". + checkNull($response).",".checkNull($resolutionRow['comment']).",". + checkNull($resStatus).")"; + + $existingResolutions[$resolutionRow['ts'].$userIdConversion[$username]] = 1; + + ## Do inserts in groups of 10 to reduce DB calls + if(count($insertValues) >= 10) { + $sql = "INSERT INTO redcap_data_quality_resolutions + (status_id,ts,user_id,response_requested,response,comment,current_query_status) + VALUES ".implode(",",$insertValues); + db_query($sql); + + if($e = db_error()) { + error_log("Data Quality Import: ".$e); + } + else { + $nextId = db_insert_id(); + for($i = 0; $i < 10; $i++) { + $resolutionInsertIds[] = $nextId + $i; + } + } + $insertValues = []; + } + } + + ## If last timestamp is after last timestamp in DB, then update main query_status + if($existingStatus && $lastTs != "" && strtotime($lastTs) > strtotime($dbTs)) { + $sql = "UPDATE redcap_data_quality_status + SET query_status = ".checkNull($resStatus)." + WHERE status_id = ".$existingStatus; + + db_query($sql); + } +} + +if(count($insertValues) > 0) { + ## Do last inserts pending in $insertValues + $sql = "INSERT INTO redcap_data_quality_resolutions + (status_id,ts,user_id,response_requested,response,comment,current_query_status) + VALUES ".implode(",",$insertValues); + db_query($sql); + + if($e = db_error()) { + error_log("Data Quality Import: ".$e); + } + else { + $nextId = db_insert_id(); + for($i = 0; $i < count($insertValues); $i++) { + $resolutionInsertIds[] = $nextId + $i; + } + } + $insertValues = []; +} + +## TODO Send other types of responses based on request +$content = json_encode($resolutionInsertIds); + +# Send the response to the requestor +// RestUtility::sendResponse(200, $content, $format); + +$dataQualityExternalModule = new \Vanderbilt\DataQualityExternalModule\DataQualityExternalModule(); +$errors = $dataQualityExternalModule->import_json_file(); + +if (!empty($errors)) +{ + require_once APP_PATH_DOCROOT . 'ProjectGeneral/header.php'; + print "
"; + foreach($errors as $error) + { + print "

$error

"; + } + print "
"; + require_once APP_PATH_DOCROOT . 'ProjectGeneral/footer.php'; +} +else +{ + header("Location: " . $dataQualityExternalModule->getUrl("index.php") . "&imported=1"); +} + +function csv($data,$headers) { + foreach($data as $dataRow) { + foreach($headers as $column => $label) { + if($label == "record") { + ## If record is blank, this must be a data resolution row + if($dataRow[$column] == "") { + + } + ## Else, this must be a status row + else { + + } + } + } + } +} diff --git a/index.php b/index.php new file mode 100644 index 0000000..76a1938 --- /dev/null +++ b/index.php @@ -0,0 +1,39 @@ + +

+ Export or Import Data Quality +

+

+ Press download to export data resolution workflow data + +
+

+ + +
+

+ +
+ +

+ +

Imported
+
+ +
+ +

Select CSV to upload:

+ + +
+

+ +
+ + + Date: Mon, 5 Oct 2020 12:25:05 -0600 Subject: [PATCH 2/8] Import json files functioning --- DataQualityExternalModule.php | 297 ------------------------- importJsonFile.php | 401 ++++++++++++++++++---------------- 2 files changed, 208 insertions(+), 490 deletions(-) diff --git a/DataQualityExternalModule.php b/DataQualityExternalModule.php index 61d81ba..b8ca015 100644 --- a/DataQualityExternalModule.php +++ b/DataQualityExternalModule.php @@ -23,301 +23,4 @@ public function getProcessJsonDownURL() { public function getProcessJsonUpURL() { return $this->getUrl("importJsonFile.php", false, false); } - - public function import_json_file() { - global $format, $returnFormat, $post; - - // Get user's user rights - $user_rights = UserRights::getPrivileges(PROJECT_ID, USERID); - $user_rights = $user_rights[PROJECT_ID][strtolower(USERID)]; - $ur = new UserRights(); - $ur->setFormLevelPrivileges(); - - $Proj = new Project(PROJECT_ID); - $errors = array(); - - // Prevent data imports for projects in inactive or archived status - if ($Proj->project['status'] > 1) { - if ($Proj->project['status'] == '2') { - $statusLabel = "Inactive"; - } elseif ($Proj->project['status'] == '3') { - $statusLabel = "Archived"; - } else { - $statusLabel = "[unknown]"; - } - array_push($errors, "Data may not be imported because the project is in $statusLabel status."); - } - - $insertValues = []; - $resolutionInsertIds = []; - $userIdConversion = array(NULL => NULL); - - $projectSettings = $Proj->project; - $dataQualityProcess = false; - $dataCommentProcess = false; - $editComments = false; - - ## Check if project has data resolution workflow turned on and ability to delete data resolution comments - if($projectSettings['data_resolution_enabled'] == 2) { - $dataQualityProcess = true; - } - else if($projectSettings['data_resolution_enabled'] == 1) { - $dataCommentProcess = true; - } - else { - array_push($errors, "Data may not be imported because the project does not have data quality enabled.")); - } - - if($projectSettings['field_comment_edit_delete'] == 1) { - $editComments = true; - } - - $single_event = false; - if(sizeof($Proj->events)==1 && $projectSettings['repeatforms']==0){ - $single_event = true; - } - - ## TODO Convert flat/csv data into nested array format - $importData = json_decode($post['import_file'],true); - - foreach($importData as $dataRow) { - $statusId = $dataRow['status_id']; - $record = $dataRow['record']; - $projectId = $dataRow['project_id']; - $eventId = $dataRow['event_id']; - $fieldName = $dataRow['field_name']; - $instance = $dataRow['instance']; - $repeatInstrument = $dataRow['repeat_instrument']; - $assignedUsername = $dataRow['assigned_username']; - $newStatusId = false; - - ## Verify field name and event ID exist on this project. Skip this row if it doesn't - if(!array_key_exists($fieldName,$Proj->metadata)) { - continue; - } - - $eventFound = false; - foreach($Proj->events as $armDetails) { - foreach($armDetails["events"] as $tempEventId => $eventDetails) { - // check for non-longitudinal project - if (count($armDetails['events']) == 1 && $single_event){ - $eventId = $tempEventId; - $eventFound = true; - break 2; - }else if($eventId == $tempEventId) { - $eventFound = true; - break 2; - } - } - } - - if(!$eventFound) { - continue; - } - - ## Skip if record/project/event/field is blank - if($record == "" || $projectId == "" || $eventId == "" || $fieldName == "") continue; - - ## Skip data where the projectId doesn't match the token's projectId - if($projectId != PROJECT_ID) continue; - - - ## Confirm if this status is already in the database - $sql = "SELECT s.status_id, s.instance, s.repeat_instrument, s.record, s.project_id, s.event_id, s.field_name - FROM redcap_data_quality_status s - WHERE ".(($statusId == "" || !is_numeric($statusId)) ? "" : "s.status_id = '".db_escape($statusId)."'")." - OR (s.record = '".db_escape($record)."' - AND s.project_id = '".db_escape($projectId)."' - AND s.event_id = '".db_escape($eventId)."' - AND s.field_name = '".db_escape($fieldName)."'". - ($repeatInstrument != NULL ? " AND s.repeat_instrument = '".db_escape($repeatInstrument)."'" : - ($instance == NULL ? " AND s.instance is NULL" : " AND s.instance = '".db_escape($instance)."'")).")"; - - $q = db_query($sql); - - ## Cache User ID to username conversion to reduce DB calls - if(!array_key_exists($assignedUsername,$userIdConversion)) { - $userIdConversion[$assignedUsername] = User::getUIIDByUsername($assignedUsername); - } - - $existingStatus = ""; - while($row = db_fetch_assoc($q)) { - ## Found a matching record/project/event/field/instance with a different status - if($row['status_id'] != $statusId) { - $existingStatus = $row['status_id']; - $newStatusId = true; - } - ## Found a row for that statusId, verify if record/project/event/field/instance matches - else if($row['record'] == $record && $row['project_id'] == $projectId - && $row['event_id'] == $eventId && $row['field_name'] == $fieldName){ - if($instance == "" || $row['instance'] == $instance) { - $existingStatus = $row['status_id']; - $newStatusId = false; - } - } - } - - ## Add new status row - if($existingStatus == "") { - $sql = "INSERT INTO redcap_data_quality_status - (non_rule,project_id,record,event_id,field_name,instance,assigned_user_id) - VALUES (1,".db_escape($projectId).",".db_escape($record).",".db_escape($eventId).",'". - db_escape($fieldName)."',".checkNull($instance).",".checkNull($userIdConversion[$assignedUsername]).")"; - - db_query($sql); - $existingStatus = db_insert_id(); - $newStatusId = true; - } - - ## Do date conversion on resolutions so they can be quickly sorted - foreach($dataRow['resolutions'] as $resKey => $resolutionRow) { - $dataRow['resolutions'][$resKey]['tsInSeconds'] = strtotime($resolutionRow['ts']); - } - - ## Sort resolutions by timestamp and get last timestamp in DB - usort($dataRow['resolutions'],function($a,$b) { - if($a['tsInSeconds'] > $b['tsInSeconds']) { - return 1; - } - else if($a['tsInSeconds'] == $b['tsInSeconds']) { - return 0; - } - return -1; - }); - - $sql = "SELECT r.ts,r.user_id - FROM redcap_data_quality_resolutions r - WHERE r.status_id = '".$existingStatus."' - ORDER BY r.ts DESC"; - - $q = db_query($sql); - $existingResolutions = array(); - while($row = db_fetch_assoc($q)) { - $existingResolutions[$row['ts'].$row['user_id']] = 1; - } - - $dbTs = db_result($q,0,"ts"); - $lastTs = ""; - - foreach($dataRow['resolutions'] as $resolutionRow) { - if($editComments) { - # TODO Allow update of data resolution status/comments - //DataQuality::editFieldComment(); - } - - // Determine the status to set - if (in_array($resolutionRow['current_query_status'], array('OPEN','CLOSED','VERIFIED','DEVERIFIED'))) { - $resStatus = $resolutionRow['current_query_status']; - } elseif ((isset($resolutionRow['response_requested']) && $resolutionRow['response_requested']) - || ($resolutionRow['response'] && $resolutionRow['response'])) { - $resStatus = 'OPEN'; - } else { - $resStatus = ''; - } - - // Make sure response is in enum list - if(in_array($resolutionRow['response'],array('DATA_MISSING','TYPOGRAPHICAL_ERROR','CONFIRMED_CORRECT','WRONG_SOURCE','OTHER'))) { - $response = $resolutionRow['response']; - } - else { - $response = ''; - } - - $resolutionId = $resolutionRow['res_id']; - $resStatusId = $resolutionRow['status_id']; - $username = $resolutionRow['username']; - - ## Skip resolution rows that don't have matching status IDs unless this is a new status - if(!$newStatusId && $resStatusId != "" && $resStatusId != $existingStatus) continue; - - ## Cache User ID to username conversion to reduce DB calls - if(!array_key_exists($username,$userIdConversion)) { - $userIdConversion[$username] = User::getUIIDByUsername($username); - } - - ## Skip resolution rows that already have a resolution from the same user for the same timestamp - ## This is to prevent duplicate response rows - if(array_key_exists($resolutionRow['ts'].$userIdConversion[$username],$existingResolutions)) { - continue; - } - - $lastTs = $resolutionRow['ts']; - - $insertValues[] = "(".checkNull($existingStatus).",".checkNull($resolutionRow['ts']).",". - checkNull($userIdConversion[$username]).",".(in_array($resolutionRow['response_requested'],[0,1]) ? $resolutionRow['response_requested'] : 0).",". - checkNull($response).",".checkNull($resolutionRow['comment']).",". - checkNull($resStatus).")"; - - $existingResolutions[$resolutionRow['ts'].$userIdConversion[$username]] = 1; - - ## Do inserts in groups of 10 to reduce DB calls - if(count($insertValues) >= 10) { - $sql = "INSERT INTO redcap_data_quality_resolutions - (status_id,ts,user_id,response_requested,response,comment,current_query_status) - VALUES ".implode(",",$insertValues); - db_query($sql); - - if($e = db_error()) { - error_log("Data Quality Import: ".$e); - } - else { - $nextId = db_insert_id(); - for($i = 0; $i < 10; $i++) { - $resolutionInsertIds[] = $nextId + $i; - } - } - $insertValues = []; - } - } - - ## If last timestamp is after last timestamp in DB, then update main query_status - if($existingStatus && $lastTs != "" && strtotime($lastTs) > strtotime($dbTs)) { - $sql = "UPDATE redcap_data_quality_status - SET query_status = ".checkNull($resStatus)." - WHERE status_id = ".$existingStatus; - - db_query($sql); - } - } - - if(count($insertValues) > 0) { - ## Do last inserts pending in $insertValues - $sql = "INSERT INTO redcap_data_quality_resolutions - (status_id,ts,user_id,response_requested,response,comment,current_query_status) - VALUES ".implode(",",$insertValues); - db_query($sql); - - if($e = db_error()) { - error_log("Data Quality Import: ".$e); - } - else { - $nextId = db_insert_id(); - for($i = 0; $i < count($insertValues); $i++) { - $resolutionInsertIds[] = $nextId + $i; - } - } - $insertValues = []; - } - - ## TODO Send other types of responses based on request - $content = json_encode($resolutionInsertIds); - return ($errors); - - function csv($data,$headers) { - foreach($data as $dataRow) { - foreach($headers as $column => $label) { - if($label == "record") { - ## If record is blank, this must be a data resolution row - if($dataRow[$column] == "") { - - } - ## Else, this must be a status row - else { - - } - } - } - } - } - } } diff --git a/importJsonFile.php b/importJsonFile.php index 01366c1..f89d4b7 100644 --- a/importJsonFile.php +++ b/importJsonFile.php @@ -39,7 +39,7 @@ $dataCommentProcess = true; } else { - array_push($errors, "Data may not be imported because the project does not have data quality enabled.")); + array_push($errors, "Data may not be imported because the project does not have data quality enabled."); } if($projectSettings['field_comment_edit_delete'] == 1) { @@ -51,251 +51,266 @@ $single_event = true; } -## TODO Convert flat/csv data into nested array format -$importData = json_decode($post['import_file'],true); - -foreach($importData as $dataRow) { - $statusId = $dataRow['status_id']; - $record = $dataRow['record']; - $projectId = $dataRow['project_id']; - $eventId = $dataRow['event_id']; - $fieldName = $dataRow['field_name']; - $instance = $dataRow['instance']; - $repeatInstrument = $dataRow['repeat_instrument']; - $assignedUsername = $dataRow['assigned_username']; - $newStatusId = false; - - ## Verify field name and event ID exist on this project. Skip this row if it doesn't - if(!array_key_exists($fieldName,$Proj->metadata)) { - continue; - } +// Import the json file +if (isset($_FILES['import_file']) && $_FILES['import_file']['error'] === UPLOAD_ERR_OK) { + $importData = json_decode(file_get_contents($_FILES['import_file']['tmp_name']),true); +}else{ + array_push($errors, "File failed to upload"); +} - $eventFound = false; - foreach($Proj->events as $armDetails) { - foreach($armDetails["events"] as $tempEventId => $eventDetails) { - // check for non-longitudinal project - if (count($armDetails['events']) == 1 && $single_event){ - $eventId = $tempEventId; - $eventFound = true; - break 2; - }else if($eventId == $tempEventId) { - $eventFound = true; - break 2; - } +if(is_null($importData)){ + array_push($errors, "Failed to decode json. Please ensure that the file selected was json..."); + $content = NULL; +}else{ + foreach($importData as $dataRow) { + $statusId = $dataRow['status_id']; + $record = $dataRow['record']; + $projectId = $dataRow['project_id']; + $eventId = $dataRow['event_id']; + $fieldName = $dataRow['field_name']; + $instance = $dataRow['instance']; + $repeatInstrument = $dataRow['repeat_instrument']; + $assignedUsername = $dataRow['assigned_username']; + $newStatusId = false; + + ## Verify field name and event ID exist on this project. Skip this row if it doesn't + if(!array_key_exists($fieldName,$Proj->metadata)) { + continue; } - } - if(!$eventFound) { - continue; - } + $eventFound = false; + foreach($Proj->events as $armDetails) { + foreach($armDetails["events"] as $tempEventId => $eventDetails) { + // check for non-longitudinal project + if (count($armDetails['events']) == 1 && $single_event){ + $eventId = $tempEventId; + $eventFound = true; + break 2; + }else if($eventId == $tempEventId) { + $eventFound = true; + break 2; + } + } + } - ## Skip if record/project/event/field is blank - if($record == "" || $projectId == "" || $eventId == "" || $fieldName == "") continue; + if(!$eventFound) { + continue; + } - ## Skip data where the projectId doesn't match the token's projectId - if($projectId != PROJECT_ID) continue; + ## Skip if record/project/event/field is blank + if($record == "" || $projectId == "" || $eventId == "" || $fieldName == "") continue; + ## Skip data where the projectId doesn't match the token's projectId + if($projectId != PROJECT_ID) continue; - ## Confirm if this status is already in the database - $sql = "SELECT s.status_id, s.instance, s.repeat_instrument, s.record, s.project_id, s.event_id, s.field_name - FROM redcap_data_quality_status s - WHERE ".(($statusId == "" || !is_numeric($statusId)) ? "" : "s.status_id = '".db_escape($statusId)."'")." - OR (s.record = '".db_escape($record)."' - AND s.project_id = '".db_escape($projectId)."' - AND s.event_id = '".db_escape($eventId)."' - AND s.field_name = '".db_escape($fieldName)."'". - ($repeatInstrument != NULL ? " AND s.repeat_instrument = '".db_escape($repeatInstrument)."'" : - ($instance == NULL ? " AND s.instance is NULL" : " AND s.instance = '".db_escape($instance)."'")).")"; - $q = db_query($sql); + ## Confirm if this status is already in the database + $sql = "SELECT s.status_id, s.instance, s.repeat_instrument, s.record, s.project_id, s.event_id, s.field_name + FROM redcap_data_quality_status s + WHERE ".(($statusId == "" || !is_numeric($statusId)) ? "" : "s.status_id = '".db_escape($statusId)."'")." + OR (s.record = '".db_escape($record)."' + AND s.project_id = '".db_escape($projectId)."' + AND s.event_id = '".db_escape($eventId)."' + AND s.field_name = '".db_escape($fieldName)."'". + ($repeatInstrument != NULL ? " AND s.repeat_instrument = '".db_escape($repeatInstrument)."'" : + ($instance == NULL ? " AND s.instance is NULL" : " AND s.instance = '".db_escape($instance)."'")).")"; - ## Cache User ID to username conversion to reduce DB calls - if(!array_key_exists($assignedUsername,$userIdConversion)) { - $userIdConversion[$assignedUsername] = User::getUIIDByUsername($assignedUsername); - } + $q = db_query($sql); - $existingStatus = ""; - while($row = db_fetch_assoc($q)) { - ## Found a matching record/project/event/field/instance with a different status - if($row['status_id'] != $statusId) { - $existingStatus = $row['status_id']; - $newStatusId = true; + ## Cache User ID to username conversion to reduce DB calls + if(!array_key_exists($assignedUsername,$userIdConversion)) { + $userIdConversion[$assignedUsername] = User::getUIIDByUsername($assignedUsername); } - ## Found a row for that statusId, verify if record/project/event/field/instance matches - else if($row['record'] == $record && $row['project_id'] == $projectId - && $row['event_id'] == $eventId && $row['field_name'] == $fieldName){ - if($instance == "" || $row['instance'] == $instance) { + + $existingStatus = ""; + while($row = db_fetch_assoc($q)) { + ## Found a matching record/project/event/field/instance with a different status + if($row['status_id'] != $statusId) { $existingStatus = $row['status_id']; - $newStatusId = false; + $newStatusId = true; + } + ## Found a row for that statusId, verify if record/project/event/field/instance matches + else if($row['record'] == $record && $row['project_id'] == $projectId + && $row['event_id'] == $eventId && $row['field_name'] == $fieldName){ + if($instance == "" || $row['instance'] == $instance) { + $existingStatus = $row['status_id']; + $newStatusId = false; + } } } - } - ## Add new status row - if($existingStatus == "") { - $sql = "INSERT INTO redcap_data_quality_status - (non_rule,project_id,record,event_id,field_name,instance,assigned_user_id) - VALUES (1,".db_escape($projectId).",".db_escape($record).",".db_escape($eventId).",'". - db_escape($fieldName)."',".checkNull($instance).",".checkNull($userIdConversion[$assignedUsername]).")"; - - db_query($sql); - $existingStatus = db_insert_id(); - $newStatusId = true; - } + ## Add new status row + if($existingStatus == "") { + $sql = "INSERT INTO redcap_data_quality_status + (non_rule,project_id,record,event_id,field_name,instance,assigned_user_id) + VALUES (1,".db_escape($projectId).",".db_escape($record).",".db_escape($eventId).",'". + db_escape($fieldName)."',".checkNull($instance).",".checkNull($userIdConversion[$assignedUsername]).")"; - ## Do date conversion on resolutions so they can be quickly sorted - foreach($dataRow['resolutions'] as $resKey => $resolutionRow) { - $dataRow['resolutions'][$resKey]['tsInSeconds'] = strtotime($resolutionRow['ts']); - } + db_query($sql); + $existingStatus = db_insert_id(); + $newStatusId = true; + } - ## Sort resolutions by timestamp and get last timestamp in DB - usort($dataRow['resolutions'],function($a,$b) { - if($a['tsInSeconds'] > $b['tsInSeconds']) { - return 1; + ## Do date conversion on resolutions so they can be quickly sorted + foreach($dataRow['resolutions'] as $resKey => $resolutionRow) { + $dataRow['resolutions'][$resKey]['tsInSeconds'] = strtotime($resolutionRow['ts']); } - else if($a['tsInSeconds'] == $b['tsInSeconds']) { - return 0; + + ## Sort resolutions by timestamp and get last timestamp in DB + usort($dataRow['resolutions'],function($a,$b) { + if($a['tsInSeconds'] > $b['tsInSeconds']) { + return 1; + } + else if($a['tsInSeconds'] == $b['tsInSeconds']) { + return 0; + } + return -1; + }); + + $sql = "SELECT r.ts,r.user_id + FROM redcap_data_quality_resolutions r + WHERE r.status_id = '".$existingStatus."' + ORDER BY r.ts DESC"; + + $q = db_query($sql); + $existingResolutions = array(); + while($row = db_fetch_assoc($q)) { + $existingResolutions[$row['ts'].$row['user_id']] = 1; } - return -1; - }); - - $sql = "SELECT r.ts,r.user_id - FROM redcap_data_quality_resolutions r - WHERE r.status_id = '".$existingStatus."' - ORDER BY r.ts DESC"; - - $q = db_query($sql); - $existingResolutions = array(); - while($row = db_fetch_assoc($q)) { - $existingResolutions[$row['ts'].$row['user_id']] = 1; - } - $dbTs = db_result($q,0,"ts"); - $lastTs = ""; + $dbTs = db_result($q,0,"ts"); + $lastTs = ""; - foreach($dataRow['resolutions'] as $resolutionRow) { - if($editComments) { - # TODO Allow update of data resolution status/comments - //DataQuality::editFieldComment(); - } + foreach($dataRow['resolutions'] as $resolutionRow) { + if($editComments) { + # TODO Allow update of data resolution status/comments + //DataQuality::editFieldComment(); + } - // Determine the status to set - if (in_array($resolutionRow['current_query_status'], array('OPEN','CLOSED','VERIFIED','DEVERIFIED'))) { - $resStatus = $resolutionRow['current_query_status']; - } elseif ((isset($resolutionRow['response_requested']) && $resolutionRow['response_requested']) - || ($resolutionRow['response'] && $resolutionRow['response'])) { - $resStatus = 'OPEN'; - } else { - $resStatus = ''; - } + // Determine the status to set + if (in_array($resolutionRow['current_query_status'], array('OPEN','CLOSED','VERIFIED','DEVERIFIED'))) { + $resStatus = $resolutionRow['current_query_status']; + } elseif ((isset($resolutionRow['response_requested']) && $resolutionRow['response_requested']) + || ($resolutionRow['response'] && $resolutionRow['response'])) { + $resStatus = 'OPEN'; + } else { + $resStatus = ''; + } - // Make sure response is in enum list - if(in_array($resolutionRow['response'],array('DATA_MISSING','TYPOGRAPHICAL_ERROR','CONFIRMED_CORRECT','WRONG_SOURCE','OTHER'))) { - $response = $resolutionRow['response']; - } - else { - $response = ''; - } + // Make sure response is in enum list + if(in_array($resolutionRow['response'],array('DATA_MISSING','TYPOGRAPHICAL_ERROR','CONFIRMED_CORRECT','WRONG_SOURCE','OTHER'))) { + $response = $resolutionRow['response']; + } + else { + $response = ''; + } - $resolutionId = $resolutionRow['res_id']; - $resStatusId = $resolutionRow['status_id']; - $username = $resolutionRow['username']; + $resolutionId = $resolutionRow['res_id']; + $resStatusId = $resolutionRow['status_id']; + $username = $resolutionRow['username']; - ## Skip resolution rows that don't have matching status IDs unless this is a new status - if(!$newStatusId && $resStatusId != "" && $resStatusId != $existingStatus) continue; + ## Skip resolution rows that don't have matching status IDs unless this is a new status + if(!$newStatusId && $resStatusId != "" && $resStatusId != $existingStatus) continue; - ## Cache User ID to username conversion to reduce DB calls - if(!array_key_exists($username,$userIdConversion)) { - $userIdConversion[$username] = User::getUIIDByUsername($username); - } + ## Cache User ID to username conversion to reduce DB calls + if(!array_key_exists($username,$userIdConversion)) { + $userIdConversion[$username] = User::getUIIDByUsername($username); + } - ## Skip resolution rows that already have a resolution from the same user for the same timestamp - ## This is to prevent duplicate response rows - if(array_key_exists($resolutionRow['ts'].$userIdConversion[$username],$existingResolutions)) { - continue; - } + ## Skip resolution rows that already have a resolution from the same user for the same timestamp + ## This is to prevent duplicate response rows + if(array_key_exists($resolutionRow['ts'].$userIdConversion[$username],$existingResolutions)) { + continue; + } - $lastTs = $resolutionRow['ts']; + $lastTs = $resolutionRow['ts']; - $insertValues[] = "(".checkNull($existingStatus).",".checkNull($resolutionRow['ts']).",". - checkNull($userIdConversion[$username]).",".(in_array($resolutionRow['response_requested'],[0,1]) ? $resolutionRow['response_requested'] : 0).",". - checkNull($response).",".checkNull($resolutionRow['comment']).",". - checkNull($resStatus).")"; + $insertValues[] = "(".checkNull($existingStatus).",".checkNull($resolutionRow['ts']).",". + checkNull($userIdConversion[$username]).",".(in_array($resolutionRow['response_requested'],[0,1]) ? $resolutionRow['response_requested'] : 0).",". + checkNull($response).",".checkNull($resolutionRow['comment']).",". + checkNull($resStatus).")"; - $existingResolutions[$resolutionRow['ts'].$userIdConversion[$username]] = 1; + $existingResolutions[$resolutionRow['ts'].$userIdConversion[$username]] = 1; - ## Do inserts in groups of 10 to reduce DB calls - if(count($insertValues) >= 10) { - $sql = "INSERT INTO redcap_data_quality_resolutions - (status_id,ts,user_id,response_requested,response,comment,current_query_status) - VALUES ".implode(",",$insertValues); - db_query($sql); + ## Do inserts in groups of 10 to reduce DB calls + if(count($insertValues) >= 10) { + $sql = "INSERT INTO redcap_data_quality_resolutions + (status_id,ts,user_id,response_requested,response,comment,current_query_status) + VALUES ".implode(",",$insertValues); + db_query($sql); - if($e = db_error()) { - error_log("Data Quality Import: ".$e); - } - else { - $nextId = db_insert_id(); - for($i = 0; $i < 10; $i++) { - $resolutionInsertIds[] = $nextId + $i; + if($e = db_error()) { + error_log("Data Quality Import: ".$e); } + else { + $nextId = db_insert_id(); + for($i = 0; $i < 10; $i++) { + $resolutionInsertIds[] = $nextId + $i; + } + } + $insertValues = []; } - $insertValues = []; } - } - ## If last timestamp is after last timestamp in DB, then update main query_status - if($existingStatus && $lastTs != "" && strtotime($lastTs) > strtotime($dbTs)) { - $sql = "UPDATE redcap_data_quality_status - SET query_status = ".checkNull($resStatus)." - WHERE status_id = ".$existingStatus; + ## If last timestamp is after last timestamp in DB, then update main query_status + if($existingStatus && $lastTs != "" && strtotime($lastTs) > strtotime($dbTs)) { + $sql = "UPDATE redcap_data_quality_status + SET query_status = ".checkNull($resStatus)." + WHERE status_id = ".$existingStatus; - db_query($sql); + db_query($sql); + } } -} -if(count($insertValues) > 0) { - ## Do last inserts pending in $insertValues - $sql = "INSERT INTO redcap_data_quality_resolutions - (status_id,ts,user_id,response_requested,response,comment,current_query_status) - VALUES ".implode(",",$insertValues); - db_query($sql); + if(count($insertValues) > 0) { + ## Do last inserts pending in $insertValues + $sql = "INSERT INTO redcap_data_quality_resolutions + (status_id,ts,user_id,response_requested,response,comment,current_query_status) + VALUES ".implode(",",$insertValues); + db_query($sql); - if($e = db_error()) { - error_log("Data Quality Import: ".$e); - } - else { - $nextId = db_insert_id(); - for($i = 0; $i < count($insertValues); $i++) { - $resolutionInsertIds[] = $nextId + $i; + if($e = db_error()) { + error_log("Data Quality Import: ".$e); + } + else { + $nextId = db_insert_id(); + for($i = 0; $i < count($insertValues); $i++) { + $resolutionInsertIds[] = $nextId + $i; + } } + $insertValues = []; } - $insertValues = []; + + ## TODO Send other types of responses based on request + $content = json_encode($resolutionInsertIds); } -## TODO Send other types of responses based on request -$content = json_encode($resolutionInsertIds); + # Send the response to the requestor // RestUtility::sendResponse(200, $content, $format); -$dataQualityExternalModule = new \Vanderbilt\DataQualityExternalModule\DataQualityExternalModule(); -$errors = $dataQualityExternalModule->import_json_file(); - if (!empty($errors)) { - require_once APP_PATH_DOCROOT . 'ProjectGeneral/header.php'; - print "
"; - foreach($errors as $error) - { - print "

$error

"; - } - print "
"; - require_once APP_PATH_DOCROOT . 'ProjectGeneral/footer.php'; + require_once APP_PATH_DOCROOT . 'ProjectGeneral/header.php'; + print "
"; + foreach($errors as $error) + { + print "

$error

"; + } + print "
"; + require_once APP_PATH_DOCROOT . 'ProjectGeneral/footer.php'; } else { - header("Location: " . $dataQualityExternalModule->getUrl("index.php") . "&imported=1"); + // $dataQualityExternalModule = new \Vanderbilt\DataQualityExternalModule\DataQualityExternalModule(); + // $errors = $dataQualityExternalModule->import_json_file(); + // header("Location: " . $dataQualityExternalModule->getUrl("index.php") . "&imported=1"); + require_once APP_PATH_DOCROOT . 'ProjectGeneral/header.php'; + print "
"; + print "

SUCCESS:$content

"; + print "
"; + require_once APP_PATH_DOCROOT . 'ProjectGeneral/footer.php'; } function csv($data,$headers) { From d3a18e19ef184d65c2da9ad517f340d665052e61 Mon Sep 17 00:00:00 2001 From: devincowan Date: Mon, 5 Oct 2020 12:40:02 -0600 Subject: [PATCH 3/8] success message --- importJsonFile.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/importJsonFile.php b/importJsonFile.php index f89d4b7..044cd60 100644 --- a/importJsonFile.php +++ b/importJsonFile.php @@ -308,7 +308,7 @@ // header("Location: " . $dataQualityExternalModule->getUrl("index.php") . "&imported=1"); require_once APP_PATH_DOCROOT . 'ProjectGeneral/header.php'; print "
"; - print "

SUCCESS:$content

"; + print "

RES IDS MODIFIED:$content

"; print "
"; require_once APP_PATH_DOCROOT . 'ProjectGeneral/footer.php'; } From 21da7ce49f3e8d2a3bd098cad312073c3ce72720 Mon Sep 17 00:00:00 2001 From: devincowan Date: Mon, 5 Oct 2020 17:31:50 -0600 Subject: [PATCH 4/8] overwrite PID on file import --- importJsonFile.php | 6 +++++- index.php | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/importJsonFile.php b/importJsonFile.php index 044cd60..4d902a2 100644 --- a/importJsonFile.php +++ b/importJsonFile.php @@ -101,7 +101,11 @@ if($record == "" || $projectId == "" || $eventId == "" || $fieldName == "") continue; ## Skip data where the projectId doesn't match the token's projectId - if($projectId != PROJECT_ID) continue; + // if($projectId != PROJECT_ID) continue; + + // Allow imports from other projects--bypassing project ID + $projectId = PROJECT_ID; + ## Confirm if this status is already in the database diff --git a/index.php b/index.php index 76a1938..5be13c9 100644 --- a/index.php +++ b/index.php @@ -25,6 +25,7 @@
Imported

+ Note PID will be ignored on import, so it is possible to import DRW data from other projects...

Select CSV to upload:

From 0acb29ceade94adffc33995a9c94ba1e5f84759d Mon Sep 17 00:00:00 2001 From: devincowan Date: Mon, 5 Oct 2020 23:01:59 -0600 Subject: [PATCH 5/8] add ability to ignore JSON items, enabling import between different projects --- importJsonFile.php | 116 +++++++++++++++++++++++++++------------------ index.php | 2 +- 2 files changed, 72 insertions(+), 46 deletions(-) diff --git a/importJsonFile.php b/importJsonFile.php index 4d902a2..7516c7a 100644 --- a/importJsonFile.php +++ b/importJsonFile.php @@ -1,5 +1,6 @@ project['status'] > 1) { @@ -63,6 +65,7 @@ $content = NULL; }else{ foreach($importData as $dataRow) { + array_push($notes, "________________________"); $statusId = $dataRow['status_id']; $record = $dataRow['record']; $projectId = $dataRow['project_id']; @@ -101,12 +104,13 @@ if($record == "" || $projectId == "" || $eventId == "" || $fieldName == "") continue; ## Skip data where the projectId doesn't match the token's projectId - // if($projectId != PROJECT_ID) continue; - - // Allow imports from other projects--bypassing project ID - $projectId = PROJECT_ID; - - + if($projectId != PROJECT_ID && is_null($ignore)){ + continue; + }else{ + // Allow imports from other projects--bypassing project ID + $ignored_project_id = $projectId; + $projectId = PROJECT_ID; + } ## Confirm if this status is already in the database $sql = "SELECT s.status_id, s.instance, s.repeat_instrument, s.record, s.project_id, s.event_id, s.field_name @@ -120,16 +124,22 @@ ($instance == NULL ? " AND s.instance is NULL" : " AND s.instance = '".db_escape($instance)."'")).")"; $q = db_query($sql); - ## Cache User ID to username conversion to reduce DB calls if(!array_key_exists($assignedUsername,$userIdConversion)) { - $userIdConversion[$assignedUsername] = User::getUIIDByUsername($assignedUsername); + if(User::getUIIDByUsername($assignedUsername) || is_null($ignore)){ + $userIdConversion[$assignedUsername] = User::getUIIDByUsername($assignedUsername); + }else{ + // Ignore the user id imported and instead overwrite with current user + $ignored_username = $username; + $userIdConversion[$assignedUsername] = User::getUIIDByUsername(USERID); + } } $existingStatus = ""; while($row = db_fetch_assoc($q)) { ## Found a matching record/project/event/field/instance with a different status if($row['status_id'] != $statusId) { + array_push($notes, "Found a matching record/project/event/field/instance with a different status"); $existingStatus = $row['status_id']; $newStatusId = true; } @@ -137,6 +147,7 @@ else if($row['record'] == $record && $row['project_id'] == $projectId && $row['event_id'] == $eventId && $row['field_name'] == $fieldName){ if($instance == "" || $row['instance'] == $instance) { + array_push($notes, "Found a row for that statusId, verify if record/project/event/field/instance matches"); $existingStatus = $row['status_id']; $newStatusId = false; } @@ -144,13 +155,17 @@ } ## Add new status row - if($existingStatus == "") { + if($existingStatus == "" || is_null($existingStatus)) { + array_push($errors,$record_esc); $sql = "INSERT INTO redcap_data_quality_status (non_rule,project_id,record,event_id,field_name,instance,assigned_user_id) - VALUES (1,".db_escape($projectId).",".db_escape($record).",".db_escape($eventId).",'". + VALUES (1,".db_escape($projectId).",'".db_escape($record)."',".db_escape($eventId).",'". db_escape($fieldName)."',".checkNull($instance).",".checkNull($userIdConversion[$assignedUsername]).")"; - - db_query($sql); + $q = db_query($sql); + if($e = db_error()) { + array_push($errors,"sql: ".$sql); + array_push($errors,"Insert Status error: ".$e); + } $existingStatus = db_insert_id(); $newStatusId = true; } @@ -186,10 +201,6 @@ $lastTs = ""; foreach($dataRow['resolutions'] as $resolutionRow) { - if($editComments) { - # TODO Allow update of data resolution status/comments - //DataQuality::editFieldComment(); - } // Determine the status to set if (in_array($resolutionRow['current_query_status'], array('OPEN','CLOSED','VERIFIED','DEVERIFIED'))) { @@ -212,26 +223,46 @@ $resolutionId = $resolutionRow['res_id']; $resStatusId = $resolutionRow['status_id']; $username = $resolutionRow['username']; + array_push($notes,"Old res_id=".$resolutionId); ## Skip resolution rows that don't have matching status IDs unless this is a new status - if(!$newStatusId && $resStatusId != "" && $resStatusId != $existingStatus) continue; + if(!$newStatusId && $resStatusId != "" && $resStatusId != $existingStatus){ + array_push($notes, "Skip resolution rows that don't have matching status IDs unless this is a new status"); + continue; + } ## Cache User ID to username conversion to reduce DB calls if(!array_key_exists($username,$userIdConversion)) { - $userIdConversion[$username] = User::getUIIDByUsername($username); + if(User::getUIIDByUsername($assignedUsername) || is_null($ignore)){ + $userIdConversion[$assignedUsername] = User::getUIIDByUsername($assignedUsername); + }else{ + // Ignore the user id imported and instead overwrite with current user + $ignored_username = $username; + $userIdConversion[$assignedUsername] = User::getUIIDByUsername(USERID); + } } ## Skip resolution rows that already have a resolution from the same user for the same timestamp ## This is to prevent duplicate response rows + # TODO: if ignoring PID, perhaps should only focus on ts and record id (not userid) if(array_key_exists($resolutionRow['ts'].$userIdConversion[$username],$existingResolutions)) { + array_push($notes,"Skip resolution rows that already have a resolution from the same user for the same timestamp"); continue; } $lastTs = $resolutionRow['ts']; + # append the userid to the comments if it will be ignored + $append_comment = $resolutionRow['comment']; + if(!is_null($ignore)){ + $append_comment .= " (DRW Import PID:".$ignored_project_id.", "; + $append_comment .= "RESID:".$resolutionId.", "; + $append_comment .= "UID:".$ignored_username.")"; + } + $insertValues[] = "(".checkNull($existingStatus).",".checkNull($resolutionRow['ts']).",". checkNull($userIdConversion[$username]).",".(in_array($resolutionRow['response_requested'],[0,1]) ? $resolutionRow['response_requested'] : 0).",". - checkNull($response).",".checkNull($resolutionRow['comment']).",". + checkNull($response).",".checkNull($append_comment).",". checkNull($resStatus).")"; $existingResolutions[$resolutionRow['ts'].$userIdConversion[$username]] = 1; @@ -264,7 +295,7 @@ db_query($sql); } - } + } # end foreach json row if(count($insertValues) > 0) { ## Do last inserts pending in $insertValues @@ -275,6 +306,7 @@ if($e = db_error()) { error_log("Data Quality Import: ".$e); + array_push($errors,$e); } else { $nextId = db_insert_id(); @@ -285,15 +317,11 @@ $insertValues = []; } - ## TODO Send other types of responses based on request $content = json_encode($resolutionInsertIds); } -# Send the response to the requestor -// RestUtility::sendResponse(200, $content, $format); - if (!empty($errors)) { require_once APP_PATH_DOCROOT . 'ProjectGeneral/header.php'; @@ -303,33 +331,31 @@ print "

$error

"; } print ""; + print "
"; + print "

RES IDS MODIFIED:$content

"; + foreach($notes as $note) + { + print "

"; + print_r($note); + print "

"; + } + print "
"; require_once APP_PATH_DOCROOT . 'ProjectGeneral/footer.php'; } else { - // $dataQualityExternalModule = new \Vanderbilt\DataQualityExternalModule\DataQualityExternalModule(); - // $errors = $dataQualityExternalModule->import_json_file(); - // header("Location: " . $dataQualityExternalModule->getUrl("index.php") . "&imported=1"); require_once APP_PATH_DOCROOT . 'ProjectGeneral/header.php'; print "
"; - print "

RES IDS MODIFIED:$content

"; - print "
"; - require_once APP_PATH_DOCROOT . 'ProjectGeneral/footer.php'; -} - -function csv($data,$headers) { - foreach($data as $dataRow) { - foreach($headers as $column => $label) { - if($label == "record") { - ## If record is blank, this must be a data resolution row - if($dataRow[$column] == "") { - - } - ## Else, this must be a status row - else { - - } - } + print "

UPLOAD COMPLETE: NO NEW RES IDS

"; + if(strlen($content) >2){ + print "

SUCCESS! RES IDS MODIFIED:$content

"; + foreach($notes as $note) + { + print "

"; + print_r($note); + print "

"; } + print ""; + require_once APP_PATH_DOCROOT . 'ProjectGeneral/footer.php'; } } diff --git a/index.php b/index.php index 5be13c9..438e93d 100644 --- a/index.php +++ b/index.php @@ -25,9 +25,9 @@
Imported

- Note PID will be ignored on import, so it is possible to import DRW data from other projects... +

Select CSV to upload:

From 337565aa38f67e26d4a666b5598e1c7a5de3b14f Mon Sep 17 00:00:00 2001 From: devincowan Date: Tue, 6 Oct 2020 09:14:08 -0600 Subject: [PATCH 6/8] small comment/notes changes --- importJsonFile.php | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/importJsonFile.php b/importJsonFile.php index 7516c7a..b665fd3 100644 --- a/importJsonFile.php +++ b/importJsonFile.php @@ -97,6 +97,7 @@ } if(!$eventFound) { + array_push($notes, "Event not found."); continue; } @@ -107,7 +108,7 @@ if($projectId != PROJECT_ID && is_null($ignore)){ continue; }else{ - // Allow imports from other projects--bypassing project ID + ## Allow imports from other projects--bypassing project ID $ignored_project_id = $projectId; $projectId = PROJECT_ID; } @@ -144,10 +145,10 @@ $newStatusId = true; } ## Found a row for that statusId, verify if record/project/event/field/instance matches - else if($row['record'] == $record && $row['project_id'] == $projectId + else if($row['record'] == $record && ($row['project_id'] == $projectId || !is_null($ignore)) && $row['event_id'] == $eventId && $row['field_name'] == $fieldName){ if($instance == "" || $row['instance'] == $instance) { - array_push($notes, "Found a row for that statusId, verify if record/project/event/field/instance matches"); + array_push($notes, "Found a row for this statusId, verified that record/project/event/field/instance matches"); $existingStatus = $row['status_id']; $newStatusId = false; } @@ -223,7 +224,7 @@ $resolutionId = $resolutionRow['res_id']; $resStatusId = $resolutionRow['status_id']; $username = $resolutionRow['username']; - array_push($notes,"Old res_id=".$resolutionId); + array_push($notes,"Old res_id=".$resolutionId."; res_status_id=".$resStatusId); ## Skip resolution rows that don't have matching status IDs unless this is a new status if(!$newStatusId && $resStatusId != "" && $resStatusId != $existingStatus){ @@ -244,9 +245,8 @@ ## Skip resolution rows that already have a resolution from the same user for the same timestamp ## This is to prevent duplicate response rows - # TODO: if ignoring PID, perhaps should only focus on ts and record id (not userid) if(array_key_exists($resolutionRow['ts'].$userIdConversion[$username],$existingResolutions)) { - array_push($notes,"Skip resolution rows that already have a resolution from the same user for the same timestamp"); + array_push($notes,"Skipping resolution row--already has a resolution from the same user for the same timestamp"); continue; } @@ -269,6 +269,7 @@ ## Do inserts in groups of 10 to reduce DB calls if(count($insertValues) >= 10) { + array_push($notes, "Inserting resolutions in batches of 10."); $sql = "INSERT INTO redcap_data_quality_resolutions (status_id,ts,user_id,response_requested,response,comment,current_query_status) VALUES ".implode(",",$insertValues); @@ -299,6 +300,7 @@ if(count($insertValues) > 0) { ## Do last inserts pending in $insertValues + array_push($notes, "Inserting final set of resolutions."); $sql = "INSERT INTO redcap_data_quality_resolutions (status_id,ts,user_id,response_requested,response,comment,current_query_status) VALUES ".implode(",",$insertValues); @@ -326,15 +328,14 @@ { require_once APP_PATH_DOCROOT . 'ProjectGeneral/header.php'; print "
"; - foreach($errors as $error) - { + foreach($errors as $error){ print "

$error

"; } print "
"; print "
"; print "

RES IDS MODIFIED:$content

"; - foreach($notes as $note) - { + print "

-------------LOG:-------------

"; + foreach($notes as $note){ print "

"; print_r($note); print "

"; @@ -346,16 +347,17 @@ { require_once APP_PATH_DOCROOT . 'ProjectGeneral/header.php'; print "
"; - print "

UPLOAD COMPLETE: NO NEW RES IDS

"; if(strlen($content) >2){ print "

SUCCESS! RES IDS MODIFIED:$content

"; - foreach($notes as $note) - { + print "

-------------LOG:-------------

"; + foreach($notes as $note){ print "

"; print_r($note); print "

"; } print "
"; require_once APP_PATH_DOCROOT . 'ProjectGeneral/footer.php'; + }else{ + print "

Process completed, but no new resolutions were added. Is that what you expected?

"; } } From 366be68c03135f256d7454bbffc2fb19ee2fe388 Mon Sep 17 00:00:00 2001 From: devincowan Date: Tue, 6 Oct 2020 11:07:08 -0600 Subject: [PATCH 7/8] add ability to save uploaded JSON to the file repository --- importJsonFile.php | 163 +++++++++++++++++++++++++++++++++++++++++++-- index.php | 1 + 2 files changed, 159 insertions(+), 5 deletions(-) diff --git a/importJsonFile.php b/importJsonFile.php index b665fd3..5cf558e 100644 --- a/importJsonFile.php +++ b/importJsonFile.php @@ -1,6 +1,7 @@ 0) { @@ -320,8 +323,14 @@ } $content = json_encode($resolutionInsertIds); -} + if(!is_null($file_repo)){ + # Save the file to the repository + array_push($notes,"Attempting to save to file repository..."); + $info_to_save = "MODIFIED CONTENT = ".implode(";",$content)."\n\n"."NOTES = ".implode(";",$notes)."\n\n"."ERRORS = ".implode(";",$errors)."\n\n"."JSON = ".json_encode($importData); + $saved = saveToFileRepository("DRW Log", $info_to_save, "txt"); + } +} if (!empty($errors)) @@ -360,4 +369,148 @@ }else{ print "

Process completed, but no new resolutions were added. Is that what you expected?

"; } + if (!empty($saved)){ + print "
NOTE FOR FILE SAVE: "; + foreach($saved as $save){ + print $save; + } + print "
"; + } +} + +/** + * Saves a file to REDCap's File Repository. Based off stolen code from BCCHR-IT/custom-template-engine + * https://github.com/BCCHR-IT/custom-template-engine + * + * @param String $filename Name of file + * @param String $file_contents Contents of file + * @param String $file_extension File extension + * @see deleteRepositoryFile() For deleting a file from the repository, if metadata failed to create. + */ +function saveToFileRepository($filename, $file_contents, $file_extension) +{ + // Upload the compiled report to the File Repository + $notes = array(); + $database_success = FALSE; + $upload_success = FALSE; + + $dummy_file_name = $filename; + $dummy_file_name = preg_replace("/[^a-zA-Z-._0-9]/","_",$dummy_file_name); + $dummy_file_name = str_replace("__","_",$dummy_file_name); + $dummy_file_name = str_replace("__","_",$dummy_file_name); + $pid = PROJECT_ID; + $uid = USERID; + + $stored_name = date('YmdHis') . "_pid" . $pid . "_" . generateRandomHash(6) . ".$file_extension"; + + $upload_success = file_put_contents(EDOC_PATH . $stored_name, $file_contents); + + if ($upload_success !== FALSE) + { + $dummy_file_size = $upload_success; + $dummy_file_type = "application/$file_extension"; + + $file_repo_name = date("Y/m/d H:i:s"); + + $sql = "INSERT INTO redcap_docs (project_id,docs_date,docs_name,docs_size,docs_type,docs_comment,docs_rights) + VALUES ($pid,CURRENT_DATE,'$dummy_file_name.$file_extension','$dummy_file_size','$dummy_file_type', + \"$file_repo_name - $filename ($uid)\",NULL)"; + + if (db_query($sql)) + { + $docs_id = db_insert_id(); + + $sql = "INSERT INTO redcap_edocs_metadata (stored_name,mime_type,doc_name,doc_size,file_extension,project_id,stored_date) + VALUES('".$stored_name."','".$dummy_file_type."','".$dummy_file_name."','".$dummy_file_size."', + '".$file_extension."','".$pid."','".date('Y-m-d H:i:s')."');"; + + if (db_query($sql)) + { + $doc_id = db_insert_id(); + $sql = "INSERT INTO redcap_docs_to_edocs (docs_id,doc_id) VALUES ('".$docs_id."','".$doc_id."');"; + + if (db_query($sql)) + { + if ($project_language == 'English') + { + // ENGLISH + $context_msg_insert = "{$lang['docs_22']} {$lang['docs_08']}"; + } + else + { + // NON-ENGLISH + $context_msg_insert = ucfirst($lang['docs_22'])." {$lang['docs_08']}"; + } + + // Logging + REDCap::logEvent("Data Quality API - Uploaded document to file repository", "Successfully uploaded $filename"); + array_push($notes,"Uploaded document to file repository"); + $context_msg = str_replace('{fetched}', '', $context_msg_insert); + $database_success = TRUE; + } + else + { + /* if this failed, we need to roll back redcap_edocs_metadata and redcap_docs */ + db_query("DELETE FROM redcap_edocs_metadata WHERE doc_id='".$doc_id."';"); + db_query("DELETE FROM redcap_docs WHERE docs_id='".$docs_id."';"); + deleteRepositoryFile($stored_name); + array_push($notes,"Upload failed: CODE1"); + } + } + else + { + /* if we failed here, we need to roll back redcap_docs */ + db_query("DELETE FROM redcap_docs WHERE docs_id='".$docs_id."';"); + deleteRepositoryFile($stored_name); + array_push($notes,"Upload failed: CODE2"); + } + } + else + { + /* if we failed here, we need to delete the file */ + deleteRepositoryFile($stored_name); + array_push($notes,"Upload failed: CODE3"); + array_push($notes, $sql); + } + }else{ + array_push($notes,"Upload failed: CODE4"); + } + + if ($database_success === FALSE) + { + $context_msg = "{$lang['global_01']}{$lang['colon']} {$lang['docs_47']}
" . $lang['docs_65'] . ' ' . maxUploadSizeFileRespository().'MB'.$lang['period']; + + if ($super_user) + { + $context_msg .= '

' . $lang['system_config_69']; + } + } + array_push($notes, $context_msg); + return $notes; +} + + +/** + * Helper function that deletes a file from the File Repository, if REDCap data about it fails + * to be inserted to the database.Stolen code from redcap version/FileRepository/index.php. + * + * @param String $file Name of file to delete + * @since 1.0 + * @access private + */ +function deleteRepositoryFile($file) +{ + global $edoc_storage_option,$wdc,$webdav_path; + if ($edoc_storage_option == '1') { + // Webdav + $wdc->delete($webdav_path . $file); + } elseif ($edoc_storage_option == '2') { + // S3 + global $amazon_s3_key, $amazon_s3_secret, $amazon_s3_bucket; + $s3 = new S3($amazon_s3_key, $amazon_s3_secret, SSL); if (isset($GLOBALS['amazon_s3_endpoint']) && $GLOBALS['amazon_s3_endpoint'] != '') $s3->setEndpoint($GLOBALS['amazon_s3_endpoint']); + $s3->deleteObject($amazon_s3_bucket, $file); + } else { + // Local + @unlink(EDOC_PATH . $file); + } } diff --git a/index.php b/index.php index 438e93d..7d1c05e 100644 --- a/index.php +++ b/index.php @@ -28,6 +28,7 @@

+

Select CSV to upload:

From 42a2bf1edc8b74013ec60a90af76c33027b9a33c Mon Sep 17 00:00:00 2001 From: devincowan Date: Tue, 12 Oct 2021 14:50:46 -0400 Subject: [PATCH 8/8] add explanation of "ignore PID..." to the Readme small wording changes to make usage more obvious --- README.md | 7 ++++++- config.json | 2 +- importJsonFile.php | 4 ++-- index.php | 18 +++++++++--------- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7d53246..5a4f54f 100644 --- a/README.md +++ b/README.md @@ -91,4 +91,9 @@ The data quality import only allows the import of new resolutions. A new status Input for data quality import is identical to the data quality export's output. Duplicate resolutions will not be imported (those with the same ts and username as existing resolutions). -Output for data quality import is a json array of [res_ids]. \ No newline at end of file +Output for data quality import is a json array of [res_ids]. + +## Syncing Data Resolutions between projects +When importing data resolutions using the index.php plugin, you will have the option to "Ignore PID and Usernames?" + +This option should be used only in situations where you are syncing resolutions between projects that have the same data dictionaries and data, but differing users and PIDs. diff --git a/config.json b/config.json index 98eb482..3a9fe2d 100644 --- a/config.json +++ b/config.json @@ -24,7 +24,7 @@ "links": { "project": [ { - "name": "Export/Import DRW JSON", + "name": "Data Quality API", "icon": "arrow_circle_double_135", "url": "index.php", "show-header-and-footer": true diff --git a/importJsonFile.php b/importJsonFile.php index 5cf558e..e36156c 100644 --- a/importJsonFile.php +++ b/importJsonFile.php @@ -328,7 +328,7 @@ # Save the file to the repository array_push($notes,"Attempting to save to file repository..."); $info_to_save = "MODIFIED CONTENT = ".implode(";",$content)."\n\n"."NOTES = ".implode(";",$notes)."\n\n"."ERRORS = ".implode(";",$errors)."\n\n"."JSON = ".json_encode($importData); - $saved = saveToFileRepository("DRW Log", $info_to_save, "txt"); + $saved = saveToFileRepository("Data Quality API Log", $info_to_save, "txt"); } } @@ -370,7 +370,7 @@ print "

Process completed, but no new resolutions were added. Is that what you expected?

"; } if (!empty($saved)){ - print "
NOTE FOR FILE SAVE: "; + print "
NOTE: "; foreach($saved as $save){ print $save; } diff --git a/index.php b/index.php index 7d1c05e..b1d52a0 100644 --- a/index.php +++ b/index.php @@ -6,15 +6,14 @@ ?>

- Export or Import Data Quality + Data Quality API

- Press download to export data resolution workflow data -
- + +

Export

- +

@@ -26,12 +25,13 @@
+

Import

-

-

-

Select CSV to upload:

+

+

+

Select JSON file to upload:

- +