diff --git a/Common/CUDA/GD_AwTV.cu b/Common/CUDA/GD_AwTV.cu index d98c13c1..13e8591e 100644 --- a/Common/CUDA/GD_AwTV.cu +++ b/Common/CUDA/GD_AwTV.cu @@ -1,713 +1,713 @@ -/*------------------------------------------------------------------------- - * - * CUDA functions for Steepest descend in POCS-type algorithms. - * - * This file will iteratively minimize by steepest descend the total variation - * of the input image, with the parameters given, using GPUs. - * - * CODE by Ander Biguri - * - * --------------------------------------------------------------------------- - * --------------------------------------------------------------------------- - * Copyright (c) 2015, University of Bath and CERN- European Organization for - * Nuclear Research - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * --------------------------------------------------------------------------- - * - * Contact: tigre.toolbox@gmail.com - * Codes : https://github.com/CERN/TIGRE - * --------------------------------------------------------------------------- - */ - - - - - - - -#define MAXTHREADS 1024 -#define MAX_BUFFER 60 - -#include "GD_AwTV.hpp" - - - - -#define cudaCheckErrors(msg) \ -do { \ - cudaError_t __err = cudaGetLastError(); \ - if (__err != cudaSuccess) { \ - mexPrintf("%s \n",msg);\ - cudaDeviceReset();\ - mexErrMsgIdAndTxt("CBCT:CUDA:GD_TV",cudaGetErrorString(__err));\ - } \ -} while (0) - -// CUDA kernels -//https://stackoverflow.com/questions/21332040/simple-cuda-kernel-optimization/21340927#21340927 - __global__ void divideArrayScalar(float* vec,float scalar,const size_t n){ - unsigned long long i = (blockIdx.x * blockDim.x) + threadIdx.x; - for(; i= 0 && z= 0 && y= 0 && x= cols || y >= rows || z >= depth ) - return; - - - float df[3] ={0.f,0.f,0.f}; - float dfi[3]={0.f,0.f,0.f}; // dfi== \partial f_{i+1,j,k} - float dfj[3]={0.f,0.f,0.f}; - float dfk[3]={0.f,0.f,0.f}; - gradient(f,df ,z ,y ,x , depth,rows,cols); - gradient(f,dfi ,z ,y ,x+1, depth,rows,cols); - gradient(f,dfj ,z ,y+1,x , depth,rows,cols); - gradient(f,dfk ,z+1,y ,x , depth,rows,cols); - float eps=0.00000001; //% avoid division by zero - - float wx=__expf(-(df[0]/delta)*(df[0]/delta)); - float wy=__expf(-(df[1]/delta)*(df[1]/delta)); - float wz=__expf(-(df[2]/delta)*(df[2]/delta)); - - float wxi=__expf(-(dfi[0]/delta)*(dfi[0]/delta)); - float wyi=__expf(-(dfi[1]/delta)*(dfi[1]/delta)); - float wzi=__expf(-(dfi[2]/delta)*(dfi[2]/delta)); - - float wxj=__expf(-(dfj[0]/delta)*(dfj[0]/delta)); - float wyj=__expf(-(dfj[1]/delta)*(dfj[1]/delta)); - float wzj=__expf(-(dfj[2]/delta)*(dfj[2]/delta)); - - float wxk=__expf(-(dfk[0]/delta)*(dfk[0]/delta)); - float wyk=__expf(-(dfk[1]/delta)*(dfk[1]/delta)); - float wzk=__expf(-(dfk[2]/delta)*(dfk[2]/delta)); - - - // this hsould do the trick I think - - dftv[idx]=(wx*df[0]+wy*df[1]+wz*df[2])/(sqrt(wx*df[0] *df[0] +wy*df[1] *df[1] +wz*df[2] *df[2])+eps) - -wzi*dfi[2]/(sqrt(wxi*dfi[0]*dfi[0]+wyi*dfi[1]*dfi[1]+wzi*dfi[2]*dfi[2]) +eps) // I wish I coudl precompute this, but if I do then Id need to recompute the gradient. - -wyj*dfj[1]/(sqrt(wxj*dfj[0]*dfj[0]+wyj*dfj[1]*dfj[1]+wzj*dfj[2]*dfj[2]) +eps) - -wxk*dfk[0]/(sqrt(wxk*dfk[0]*dfk[0]+wyk*dfk[1]*dfk[1]+wzk*dfk[2]*dfk[2]) +eps); - - - return; - - } - - __device__ void warpReduce(volatile float *sdata, size_t tid) { - sdata[tid] += sdata[tid + 32]; - sdata[tid] += sdata[tid + 16]; - sdata[tid] += sdata[tid + 8]; - sdata[tid] += sdata[tid + 4]; - sdata[tid] += sdata[tid + 2]; - sdata[tid] += sdata[tid + 1]; - } - - __global__ void reduceNorm2(float *g_idata, float *g_odata, size_t n){ - extern __shared__ volatile float sdata[]; - //http://stackoverflow.com/a/35133396/1485872 - size_t tid = threadIdx.x; - size_t i = blockIdx.x*blockDim.x + tid; - size_t gridSize = blockDim.x*gridDim.x; - float mySum = 0; - float value=0; - while (i < n) { - value=g_idata[i]; //avoid reading twice - mySum += value*value; - i += gridSize; - } - sdata[tid] = mySum; - __syncthreads(); - - if (tid < 512) - sdata[tid] += sdata[tid + 512]; - __syncthreads(); - if (tid < 256) - sdata[tid] += sdata[tid + 256]; - __syncthreads(); - - if (tid < 128) - sdata[tid] += sdata[tid + 128]; - __syncthreads(); - - if (tid < 64) - sdata[tid] += sdata[tid + 64]; - __syncthreads(); - - -#if (__CUDART_VERSION >= 9000) - if ( tid < 32 ) - { - mySum = sdata[tid] + sdata[tid + 32]; - for (int offset = warpSize/2; offset > 0; offset /= 2) { - mySum += __shfl_down_sync(0xFFFFFFFF, mySum, offset,32); - } - } -#else - if (tid < 32) { - warpReduce(sdata, tid); - mySum = sdata[0]; - } -#endif - if (tid == 0) g_odata[blockIdx.x] = mySum; - } - - __global__ void reduceSum(float *g_idata, float *g_odata, size_t n){ - extern __shared__ volatile float sdata[]; - //http://stackoverflow.com/a/35133396/1485872 - size_t tid = threadIdx.x; - size_t i = blockIdx.x*blockDim.x + tid; - size_t gridSize = blockDim.x*gridDim.x; - float mySum = 0; - // float value=0; - while (i < n) { - mySum += g_idata[i]; - i += gridSize; - } - sdata[tid] = mySum; - __syncthreads(); - - if (tid < 512) - sdata[tid] += sdata[tid + 512]; - __syncthreads(); - if (tid < 256) - sdata[tid] += sdata[tid + 256]; - __syncthreads(); - - if (tid < 128) - sdata[tid] += sdata[tid + 128]; - __syncthreads(); - - if (tid < 64) - sdata[tid] += sdata[tid + 64]; - __syncthreads(); - - -#if (__CUDART_VERSION >= 9000) - if ( tid < 32 ) - { - mySum = sdata[tid] + sdata[tid + 32]; - for (int offset = warpSize/2; offset > 0; offset /= 2) { - mySum += __shfl_down_sync(0xFFFFFFFF, mySum, offset,32); - } - } -#else - if (tid < 32) { - warpReduce(sdata, tid); - mySum = sdata[0]; - } -#endif - if (tid == 0) g_odata[blockIdx.x] = mySum; - } - - - - -// main function -void aw_pocs_tv(float* img,float* dst,float alpha,const long* image_size, int maxIter,const float delta, const GpuIds& gpuids){ - // Prepare for MultiGPU - int deviceCount = gpuids.GetLength(); - cudaCheckErrors("Device query fail"); - if (deviceCount == 0) { - mexErrMsgIdAndTxt("minimizeAwTV:GD_AwTV:GPUselect","There are no available device(s) that support CUDA\n"); - } - // - // CODE assumes - // 1.-All available devices are usable by this code - // 2.-All available devices are equal, they are the same machine (warning thrown) - // Check the available devices, and if they are the same - if (!gpuids.AreEqualDevices()) { - mexWarnMsgIdAndTxt("minimizeAwTV:GD_AwTV:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); - } - int dev; - - // We don't know if the devices are being used. lets check that. and only use the amount of memory we need. - // check free memory - size_t mem_GPU_global; - checkFreeMemory(gpuids, &mem_GPU_global); - - - - // %5 of free memory should be enough, we have almost no variables in these kernels - size_t total_pixels = image_size[0] * image_size[1] * image_size[2] ; - size_t mem_slice_image = sizeof(float)* image_size[0] * image_size[1] ; - size_t mem_size_image = sizeof(float)* total_pixels; - size_t mem_auxiliary = sizeof(float)* (total_pixels + MAXTHREADS - 1) / MAXTHREADS; - - // Decide how are we handling the distribution of computation - size_t mem_img_each_GPU; - - unsigned int buffer_length=2; - //Does everything fit in the GPU? - unsigned int slices_per_split; - - // if it is a thin problem (no need to split), just use one GPU - if (image_size[2]<4){deviceCount=1;} - - unsigned int splits=1; // if the number does not fit in an uint, you have more serious trouble than this. - if(mem_GPU_global> 3*mem_size_image+3*(deviceCount-1)*mem_slice_image*buffer_length+mem_auxiliary) { - // We only need to split if we have extra GPUs - slices_per_split=(image_size[2]+deviceCount-1)/deviceCount; - mem_img_each_GPU=mem_slice_image*((slices_per_split+buffer_length*2)); - }else{ - // As mem_auxiliary is not expected to be a large value (for a 2000^3 image is around 28Mbytes), lets for now assume we need it all - size_t mem_free=mem_GPU_global-mem_auxiliary; - - splits=(unsigned int)(ceil(((float)(3*mem_size_image)/(float)(deviceCount))/mem_free)); - // Now, there is an overhead here, as each splits should have 2 slices more, to account for overlap of images. - // lets make sure these 2 slices fit, if they do not, add 1 to splits. - slices_per_split=(image_size[2]+deviceCount*splits-1)/(deviceCount*splits); - mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); - - // if the new stuff does not fit in the GPU, it means we are in the edge case where adding that extra slice will overflow memory - if (mem_GPU_global< 3*mem_img_each_GPU+mem_auxiliary){ - // one more split should do the job, as its an edge case. - splits++; - //recompute for later - slices_per_split=(image_size[2]+deviceCount*splits-1)/(deviceCount*splits); // amount of slices that fit on a GPU. Later we add 2 to these, as we need them for overlap - mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); - } - - - // How many EXTRA buffer slices should be able to fit in here??!?! - // Only do it if there are splits needed. - if(splits>1){ - mem_free=mem_GPU_global-(3*mem_img_each_GPU+mem_auxiliary); - unsigned int extra_buff=(mem_free/mem_slice_image); - buffer_length=(extra_buff/2)/3; // we need double whatever this results in, rounded down. - buffer_length=max(buffer_length,2);// minimum 2 - buffer_length=min(MAX_BUFFER,buffer_length); - - mem_img_each_GPU=mem_slice_image*(slices_per_split+buffer_length*2); - - }else{ - buffer_length=2; - } - - // Assert - if (mem_GPU_global< 3*mem_img_each_GPU+mem_auxiliary){ - mexErrMsgIdAndTxt("minimizeAwTV:GD_AwTV:GPU","Assertion Failed. Logic behind splitting flawed! Please tell: ander.biguri@gmail.com\n"); - } - } - - - // Assert - - if ((slices_per_split+buffer_length*2)*image_size[0]*image_size[1]* sizeof(float)!= mem_img_each_GPU){ - mexErrMsgIdAndTxt("minimizeAwTV:GD_AwTV:GPU","Assertion Failed. Memory needed calculation broken! Please tell: ander.biguri@gmail.com\n"); - } - - - - - - - float** d_image= (float**)malloc(deviceCount*sizeof(float*)); - float** d_dimgTV= (float**)malloc(deviceCount*sizeof(float*)); - float** d_norm2aux= (float**)malloc(deviceCount*sizeof(float*)); - float** d_norm2= (float**)malloc(deviceCount*sizeof(float*)); - - // allocate memory in each GPU - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - - cudaMalloc((void**)&d_image[dev] , mem_img_each_GPU); - cudaMemset( d_image[dev],0 , mem_img_each_GPU); - cudaMalloc((void**)&d_dimgTV[dev] , mem_img_each_GPU); - cudaMemset( d_dimgTV[dev],0 , mem_img_each_GPU); - cudaMalloc((void**)&d_norm2[dev] , slices_per_split*mem_slice_image); - cudaMemset( d_norm2[dev],0 , slices_per_split*mem_slice_image); - cudaMalloc((void**)&d_norm2aux[dev] , mem_auxiliary); - cudaMemset( d_norm2aux[dev],0 , mem_auxiliary); - cudaCheckErrors("Malloc error"); - - - } - unsigned long long buffer_pixels=buffer_length*image_size[0]*image_size[1]; - float* buffer; - if(splits>1){ - mexWarnMsgIdAndTxt("minimizeAwTV:GD_AwTV:Image_split","Your image can not be fully split between the available GPUs. The computation of minTV will be significantly slowed due to the image size.\nApproximated mathematics turned on for computational speed."); - }else{ - cudaMallocHost((void**)&buffer,buffer_length*image_size[0]*image_size[1]*sizeof(float)); - } - - - - // Lets try to make the host memory pinned: - // We laredy queried the GPU and assuemd they are the same, thus should have the same attributes. - int isHostRegisterSupported = 0; -#if CUDART_VERSION >= 9020 - cudaDeviceGetAttribute(&isHostRegisterSupported,cudaDevAttrHostRegisterSupported,gpuids[0]); -#endif - // splits>2 is completely empirical observation - if (isHostRegisterSupported & splits>2){ - cudaHostRegister(img ,image_size[2]*image_size[1]*image_size[0]*sizeof(float),cudaHostRegisterPortable); - cudaHostRegister(dst ,image_size[2]*image_size[1]*image_size[0]*sizeof(float),cudaHostRegisterPortable); - } - cudaCheckErrors("Error pinning memory"); - - - - // Create streams - int nStream_device=2; - int nStreams=deviceCount*nStream_device; - cudaStream_t* stream=(cudaStream_t*)malloc(nStreams*sizeof(cudaStream_t)); - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - for (int i = 0; i < nStream_device; ++i){ - cudaStreamCreate(&stream[i+dev*nStream_device]); - } - } - cudaCheckErrors("Stream creation fail"); - - - // For the reduction - - double totalsum_prev; - double totalsum; - float sum_curr_spl; - float * sumnorm2; - cudaMallocHost((void**)&sumnorm2,deviceCount*sizeof(float)); - - unsigned int curr_slices; - unsigned long long curr_pixels; - size_t linear_idx_start; - unsigned long long* offset_device=(unsigned long long*)malloc(deviceCount*sizeof(unsigned long long)); - unsigned long long* offset_host =(unsigned long long*)malloc(deviceCount*sizeof(unsigned long long)); - unsigned long long* bytes_device =(unsigned long long*)malloc(deviceCount*sizeof(unsigned long long)); - bool is_first_chunk; - bool is_last_chunk; - for(unsigned int i=0;i1){ - totalsum_prev=0; - } - for(unsigned int sp=0;sp1 & i>0){ - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaMemcpyAsync(d_image[dev]+offset_device[dev], dst+offset_host[dev] , bytes_device[dev]*sizeof(float), cudaMemcpyHostToDevice,stream[dev*nStream_device+1]); - - - } - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - } - } - cudaCheckErrors("Memcpy failure on multi split"); - - for(unsigned int ib=0; (ib<(buffer_length-1)) && ((i+ib)>>(d_image[dev],d_dimgTV[dev],(long)(curr_slices+buffer_length*2-1), image_size[1],image_size[0],delta); - - } - - - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - curr_slices=((sp*deviceCount+dev+1)*slices_per_split> >(d_norm2[dev], d_norm2aux[dev], total_pixels); - - } - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - curr_slices=((sp*deviceCount+dev+1)*slices_per_split 1) { - reduceSum << <1, dimblockRed, MAXTHREADS*sizeof(float),stream[dev*nStream_device] >> >(d_norm2aux[dev], d_norm2[dev], dimgridRed); - cudaStreamSynchronize(stream[dev*nStream_device]); - cudaMemcpyAsync(&sumnorm2[dev], d_norm2[dev], sizeof(float), cudaMemcpyDeviceToHost,stream[dev*nStream_device+1]); - } - else { - cudaStreamSynchronize(stream[dev*nStream_device]); - cudaMemcpyAsync(&sumnorm2[dev], d_norm2aux[dev], sizeof(float), cudaMemcpyDeviceToHost,stream[dev*nStream_device+1]); - } - } - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - } - cudaCheckErrors("Reduction error"); - - - // Accumulate the norm accross devices - sum_curr_spl=0; - // this is CPU code - for (dev = 0; dev < deviceCount; dev++){ - sum_curr_spl+=sumnorm2[dev]; - } - sum_curr_spl+=0.0000001f; // avoid division by zero - - // If we have more than one splits, lets use the result from prior calls - if(i>0 && splits>1){ - // this is already stored: - //totalsum=totalsum_prev; - }else{ - totalsum=sum_curr_spl; - } - - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - curr_slices=((sp*deviceCount+dev+1)*slices_per_split>>(d_dimgTV[dev]+buffer_pixels,(float)sqrt(totalsum),total_pixels); - //MULTIPLY HYPERPARAMETER - multiplyArrayScalar<<<60,MAXTHREADS,0,stream[dev*nStream_device]>>>(d_dimgTV[dev]+buffer_pixels,alpha, total_pixels); - } - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - } - cudaCheckErrors("Scalar operations error"); - - //SUBSTRACT GRADIENT - ////////////////////////////////////////////// - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - curr_slices=((sp*deviceCount+dev+1)*slices_per_split>>(d_image[dev]+buffer_pixels,d_dimgTV[dev]+buffer_pixels, total_pixels); - } - } - - // Synchronize mathematics, make sure bounding pixels are correct - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - } - - if(splits==1){ - for(dev=0; dev0){ - cudaSetDevice(gpuids[dev-1]); - cudaMemcpyAsync(buffer, d_image[dev-1]+total_pixels+buffer_pixels, buffer_pixels*sizeof(float), cudaMemcpyDeviceToHost); - cudaSetDevice(gpuids[dev]); - cudaMemcpyAsync(d_image[dev],buffer, buffer_pixels*sizeof(float), cudaMemcpyHostToDevice); - } - } - }else{ - - // We need to take it out :( - for(dev=0; dev2){ - cudaHostUnregister(img); - cudaHostUnregister(dst); - } - for (int i = 0; i < nStreams; ++i) - cudaStreamDestroy(stream[i]) ; - cudaCheckErrors("Memory free"); -// cudaDeviceReset(); - } - -void checkFreeMemory(const GpuIds& gpuids, size_t *mem_GPU_global){ - size_t memfree; - size_t memtotal; - const int deviceCount = gpuids.GetLength(); - for (int dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaMemGetInfo(&memfree,&memtotal); - if(dev==0) *mem_GPU_global=memfree; - if(memfree= 0 && z= 0 && y= 0 && x= cols || y >= rows || z >= depth ) + return; + + + float df[3] ={0.f,0.f,0.f}; + float dfi[3]={0.f,0.f,0.f}; // dfi== \partial f_{i+1,j,k} + float dfj[3]={0.f,0.f,0.f}; + float dfk[3]={0.f,0.f,0.f}; + gradient(f,df ,z ,y ,x , depth,rows,cols); + gradient(f,dfi ,z ,y ,x+1, depth,rows,cols); + gradient(f,dfj ,z ,y+1,x , depth,rows,cols); + gradient(f,dfk ,z+1,y ,x , depth,rows,cols); + float eps=0.00000001; //% avoid division by zero + + float wx=__expf(-(df[0]/delta)*(df[0]/delta)); + float wy=__expf(-(df[1]/delta)*(df[1]/delta)); + float wz=__expf(-(df[2]/delta)*(df[2]/delta)); + + float wxi=__expf(-(dfi[0]/delta)*(dfi[0]/delta)); + float wyi=__expf(-(dfi[1]/delta)*(dfi[1]/delta)); + float wzi=__expf(-(dfi[2]/delta)*(dfi[2]/delta)); + + float wxj=__expf(-(dfj[0]/delta)*(dfj[0]/delta)); + float wyj=__expf(-(dfj[1]/delta)*(dfj[1]/delta)); + float wzj=__expf(-(dfj[2]/delta)*(dfj[2]/delta)); + + float wxk=__expf(-(dfk[0]/delta)*(dfk[0]/delta)); + float wyk=__expf(-(dfk[1]/delta)*(dfk[1]/delta)); + float wzk=__expf(-(dfk[2]/delta)*(dfk[2]/delta)); + + + // this hsould do the trick I think + + dftv[idx]=(wx*df[0]+wy*df[1]+wz*df[2])/(sqrt(wx*df[0] *df[0] +wy*df[1] *df[1] +wz*df[2] *df[2])+eps) + -wzi*dfi[2]/(sqrt(wxi*dfi[0]*dfi[0]+wyi*dfi[1]*dfi[1]+wzi*dfi[2]*dfi[2]) +eps) // I wish I coudl precompute this, but if I do then Id need to recompute the gradient. + -wyj*dfj[1]/(sqrt(wxj*dfj[0]*dfj[0]+wyj*dfj[1]*dfj[1]+wzj*dfj[2]*dfj[2]) +eps) + -wxk*dfk[0]/(sqrt(wxk*dfk[0]*dfk[0]+wyk*dfk[1]*dfk[1]+wzk*dfk[2]*dfk[2]) +eps); + + + return; + + } + + __device__ void warpReduce(volatile float *sdata, size_t tid) { + sdata[tid] += sdata[tid + 32]; + sdata[tid] += sdata[tid + 16]; + sdata[tid] += sdata[tid + 8]; + sdata[tid] += sdata[tid + 4]; + sdata[tid] += sdata[tid + 2]; + sdata[tid] += sdata[tid + 1]; + } + + __global__ void reduceNorm2(float *g_idata, float *g_odata, size_t n){ + extern __shared__ volatile float sdata[]; + //http://stackoverflow.com/a/35133396/1485872 + size_t tid = threadIdx.x; + size_t i = blockIdx.x*blockDim.x + tid; + size_t gridSize = blockDim.x*gridDim.x; + float mySum = 0; + float value=0; + while (i < n) { + value=g_idata[i]; //avoid reading twice + mySum += value*value; + i += gridSize; + } + sdata[tid] = mySum; + __syncthreads(); + + if (tid < 512) + sdata[tid] += sdata[tid + 512]; + __syncthreads(); + if (tid < 256) + sdata[tid] += sdata[tid + 256]; + __syncthreads(); + + if (tid < 128) + sdata[tid] += sdata[tid + 128]; + __syncthreads(); + + if (tid < 64) + sdata[tid] += sdata[tid + 64]; + __syncthreads(); + + +#if (__CUDART_VERSION >= 9000) + if ( tid < 32 ) + { + mySum = sdata[tid] + sdata[tid + 32]; + for (int offset = warpSize/2; offset > 0; offset /= 2) { + mySum += __shfl_down_sync(0xFFFFFFFF, mySum, offset,32); + } + } +#else + if (tid < 32) { + warpReduce(sdata, tid); + mySum = sdata[0]; + } +#endif + if (tid == 0) g_odata[blockIdx.x] = mySum; + } + + __global__ void reduceSum(float *g_idata, float *g_odata, size_t n){ + extern __shared__ volatile float sdata[]; + //http://stackoverflow.com/a/35133396/1485872 + size_t tid = threadIdx.x; + size_t i = blockIdx.x*blockDim.x + tid; + size_t gridSize = blockDim.x*gridDim.x; + float mySum = 0; + // float value=0; + while (i < n) { + mySum += g_idata[i]; + i += gridSize; + } + sdata[tid] = mySum; + __syncthreads(); + + if (tid < 512) + sdata[tid] += sdata[tid + 512]; + __syncthreads(); + if (tid < 256) + sdata[tid] += sdata[tid + 256]; + __syncthreads(); + + if (tid < 128) + sdata[tid] += sdata[tid + 128]; + __syncthreads(); + + if (tid < 64) + sdata[tid] += sdata[tid + 64]; + __syncthreads(); + + +#if (__CUDART_VERSION >= 9000) + if ( tid < 32 ) + { + mySum = sdata[tid] + sdata[tid + 32]; + for (int offset = warpSize/2; offset > 0; offset /= 2) { + mySum += __shfl_down_sync(0xFFFFFFFF, mySum, offset,32); + } + } +#else + if (tid < 32) { + warpReduce(sdata, tid); + mySum = sdata[0]; + } +#endif + if (tid == 0) g_odata[blockIdx.x] = mySum; + } + + + + +// main function +void aw_pocs_tv(float* img,float* dst,float alpha,const long* image_size, int maxIter,const float delta, const GpuIds& gpuids){ + // Prepare for MultiGPU + int deviceCount = gpuids.GetLength(); + hipCheckErrors("Device query fail"); + if (deviceCount == 0) { + mexErrMsgIdAndTxt("minimizeAwTV:GD_AwTV:GPUselect","There are no available device(s) that support CUDA\n"); + } + // + // CODE assumes + // 1.-All available devices are usable by this code + // 2.-All available devices are equal, they are the same machine (warning thrown) + // Check the available devices, and if they are the same + if (!gpuids.AreEqualDevices()) { + mexWarnMsgIdAndTxt("minimizeAwTV:GD_AwTV:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); + } + int dev; + + // We don't know if the devices are being used. lets check that. and only use the amount of memory we need. + // check free memory + size_t mem_GPU_global; + checkFreeMemory(gpuids, &mem_GPU_global); + + + + // %5 of free memory should be enough, we have almost no variables in these kernels + size_t total_pixels = image_size[0] * image_size[1] * image_size[2] ; + size_t mem_slice_image = sizeof(float)* image_size[0] * image_size[1] ; + size_t mem_size_image = sizeof(float)* total_pixels; + size_t mem_auxiliary = sizeof(float)* (total_pixels + MAXTHREADS - 1) / MAXTHREADS; + + // Decide how are we handling the distribution of computation + size_t mem_img_each_GPU; + + unsigned int buffer_length=2; + //Does everything fit in the GPU? + unsigned int slices_per_split; + + // if it is a thin problem (no need to split), just use one GPU + if (image_size[2]<4){deviceCount=1;} + + unsigned int splits=1; // if the number does not fit in an uint, you have more serious trouble than this. + if(mem_GPU_global> 3*mem_size_image+3*(deviceCount-1)*mem_slice_image*buffer_length+mem_auxiliary) { + // We only need to split if we have extra GPUs + slices_per_split=(image_size[2]+deviceCount-1)/deviceCount; + mem_img_each_GPU=mem_slice_image*((slices_per_split+buffer_length*2)); + }else{ + // As mem_auxiliary is not expected to be a large value (for a 2000^3 image is around 28Mbytes), lets for now assume we need it all + size_t mem_free=mem_GPU_global-mem_auxiliary; + + splits=(unsigned int)(ceil(((float)(3*mem_size_image)/(float)(deviceCount))/mem_free)); + // Now, there is an overhead here, as each splits should have 2 slices more, to account for overlap of images. + // lets make sure these 2 slices fit, if they do not, add 1 to splits. + slices_per_split=(image_size[2]+deviceCount*splits-1)/(deviceCount*splits); + mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); + + // if the new stuff does not fit in the GPU, it means we are in the edge case where adding that extra slice will overflow memory + if (mem_GPU_global< 3*mem_img_each_GPU+mem_auxiliary){ + // one more split should do the job, as its an edge case. + splits++; + //recompute for later + slices_per_split=(image_size[2]+deviceCount*splits-1)/(deviceCount*splits); // amount of slices that fit on a GPU. Later we add 2 to these, as we need them for overlap + mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); + } + + + // How many EXTRA buffer slices should be able to fit in here??!?! + // Only do it if there are splits needed. + if(splits>1){ + mem_free=mem_GPU_global-(3*mem_img_each_GPU+mem_auxiliary); + unsigned int extra_buff=(mem_free/mem_slice_image); + buffer_length=(extra_buff/2)/3; // we need double whatever this results in, rounded down. + buffer_length=max(buffer_length,2);// minimum 2 + buffer_length=min(MAX_BUFFER,buffer_length); + + mem_img_each_GPU=mem_slice_image*(slices_per_split+buffer_length*2); + + }else{ + buffer_length=2; + } + + // Assert + if (mem_GPU_global< 3*mem_img_each_GPU+mem_auxiliary){ + mexErrMsgIdAndTxt("minimizeAwTV:GD_AwTV:GPU","Assertion Failed. Logic behind splitting flawed! Please tell: ander.biguri@gmail.com\n"); + } + } + + + // Assert + + if ((slices_per_split+buffer_length*2)*image_size[0]*image_size[1]* sizeof(float)!= mem_img_each_GPU){ + mexErrMsgIdAndTxt("minimizeAwTV:GD_AwTV:GPU","Assertion Failed. Memory needed calculation broken! Please tell: ander.biguri@gmail.com\n"); + } + + + + + + + float** d_image= (float**)malloc(deviceCount*sizeof(float*)); + float** d_dimgTV= (float**)malloc(deviceCount*sizeof(float*)); + float** d_norm2aux= (float**)malloc(deviceCount*sizeof(float*)); + float** d_norm2= (float**)malloc(deviceCount*sizeof(float*)); + + // allocate memory in each GPU + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + + hipMalloc((void**)&d_image[dev] , mem_img_each_GPU); + hipMemset( d_image[dev],0 , mem_img_each_GPU); + hipMalloc((void**)&d_dimgTV[dev] , mem_img_each_GPU); + hipMemset( d_dimgTV[dev],0 , mem_img_each_GPU); + hipMalloc((void**)&d_norm2[dev] , slices_per_split*mem_slice_image); + hipMemset( d_norm2[dev],0 , slices_per_split*mem_slice_image); + hipMalloc((void**)&d_norm2aux[dev] , mem_auxiliary); + hipMemset( d_norm2aux[dev],0 , mem_auxiliary); + hipCheckErrors("Malloc error"); + + + } + unsigned long long buffer_pixels=buffer_length*image_size[0]*image_size[1]; + float* buffer; + if(splits>1){ + mexWarnMsgIdAndTxt("minimizeAwTV:GD_AwTV:Image_split","Your image can not be fully split between the available GPUs. The computation of minTV will be significantly slowed due to the image size.\nApproximated mathematics turned on for computational speed."); + }else{ + hipMallocHost((void**)&buffer,buffer_length*image_size[0]*image_size[1]*sizeof(float)); + } + + + + // Lets try to make the host memory pinned: + // We laredy queried the GPU and assuemd they are the same, thus should have the same attributes. + int isHostRegisterSupported = 0; +#if CUDART_VERSION >= 9020 + hipDeviceGetAttribute(&isHostRegisterSupported,hipDevAttrHostRegisterSupported,gpuids[0]); +#endif + // splits>2 is completely empirical observation + if (isHostRegisterSupported & splits>2){ + hipHostRegister(img ,image_size[2]*image_size[1]*image_size[0]*sizeof(float),hipHostRegisterPortable); + hipHostRegister(dst ,image_size[2]*image_size[1]*image_size[0]*sizeof(float),hipHostRegisterPortable); + } + hipCheckErrors("Error pinning memory"); + + + + // Create streams + int nStream_device=2; + int nStreams=deviceCount*nStream_device; + hipStream_t* stream=(hipStream_t*)malloc(nStreams*sizeof(hipStream_t)); + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + for (int i = 0; i < nStream_device; ++i){ + hipStreamCreate(&stream[i+dev*nStream_device]); + } + } + hipCheckErrors("Stream creation fail"); + + + // For the reduction + + double totalsum_prev; + double totalsum; + float sum_curr_spl; + float * sumnorm2; + hipMallocHost((void**)&sumnorm2,deviceCount*sizeof(float)); + + unsigned int curr_slices; + unsigned long long curr_pixels; + size_t linear_idx_start; + unsigned long long* offset_device=(unsigned long long*)malloc(deviceCount*sizeof(unsigned long long)); + unsigned long long* offset_host =(unsigned long long*)malloc(deviceCount*sizeof(unsigned long long)); + unsigned long long* bytes_device =(unsigned long long*)malloc(deviceCount*sizeof(unsigned long long)); + bool is_first_chunk; + bool is_last_chunk; + for(unsigned int i=0;i1){ + totalsum_prev=0; + } + for(unsigned int sp=0;sp1 & i>0){ + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipMemcpyAsync(d_image[dev]+offset_device[dev], dst+offset_host[dev] , bytes_device[dev]*sizeof(float), hipMemcpyHostToDevice,stream[dev*nStream_device+1]); + + + } + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + } + } + hipCheckErrors("Memcpy failure on multi split"); + + for(unsigned int ib=0; (ib<(buffer_length-1)) && ((i+ib)>>(d_image[dev],d_dimgTV[dev],(long)(curr_slices+buffer_length*2-1), image_size[1],image_size[0],delta); + + } + + + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + curr_slices=((sp*deviceCount+dev+1)*slices_per_split> >(d_norm2[dev], d_norm2aux[dev], total_pixels); + + } + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + curr_slices=((sp*deviceCount+dev+1)*slices_per_split 1) { + reduceSum << <1, dimblockRed, MAXTHREADS*sizeof(float),stream[dev*nStream_device] >> >(d_norm2aux[dev], d_norm2[dev], dimgridRed); + hipStreamSynchronize(stream[dev*nStream_device]); + hipMemcpyAsync(&sumnorm2[dev], d_norm2[dev], sizeof(float), hipMemcpyDeviceToHost,stream[dev*nStream_device+1]); + } + else { + hipStreamSynchronize(stream[dev*nStream_device]); + hipMemcpyAsync(&sumnorm2[dev], d_norm2aux[dev], sizeof(float), hipMemcpyDeviceToHost,stream[dev*nStream_device+1]); + } + } + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + } + hipCheckErrors("Reduction error"); + + + // Accumulate the norm accross devices + sum_curr_spl=0; + // this is CPU code + for (dev = 0; dev < deviceCount; dev++){ + sum_curr_spl+=sumnorm2[dev]; + } + sum_curr_spl+=0.0000001f; // avoid division by zero + + // If we have more than one splits, lets use the result from prior calls + if(i>0 && splits>1){ + // this is already stored: + //totalsum=totalsum_prev; + }else{ + totalsum=sum_curr_spl; + } + + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + curr_slices=((sp*deviceCount+dev+1)*slices_per_split>>(d_dimgTV[dev]+buffer_pixels,(float)sqrt(totalsum),total_pixels); + //MULTIPLY HYPERPARAMETER + multiplyArrayScalar<<<60,MAXTHREADS,0,stream[dev*nStream_device]>>>(d_dimgTV[dev]+buffer_pixels,alpha, total_pixels); + } + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + } + hipCheckErrors("Scalar operations error"); + + //SUBSTRACT GRADIENT + ////////////////////////////////////////////// + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + curr_slices=((sp*deviceCount+dev+1)*slices_per_split>>(d_image[dev]+buffer_pixels,d_dimgTV[dev]+buffer_pixels, total_pixels); + } + } + + // Synchronize mathematics, make sure bounding pixels are correct + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + } + + if(splits==1){ + for(dev=0; dev0){ + hipSetDevice(gpuids[dev-1]); + hipMemcpyAsync(buffer, d_image[dev-1]+total_pixels+buffer_pixels, buffer_pixels*sizeof(float), hipMemcpyDeviceToHost); + hipSetDevice(gpuids[dev]); + hipMemcpyAsync(d_image[dev],buffer, buffer_pixels*sizeof(float), hipMemcpyHostToDevice); + } + } + }else{ + + // We need to take it out :( + for(dev=0; dev2){ + hipHostUnregister(img); + hipHostUnregister(dst); + } + for (int i = 0; i < nStreams; ++i) + hipStreamDestroy(stream[i]) ; + hipCheckErrors("Memory free"); +// hipDeviceReset(); + } + +void checkFreeMemory(const GpuIds& gpuids, size_t *mem_GPU_global){ + size_t memfree; + size_t memtotal; + const int deviceCount = gpuids.GetLength(); + for (int dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipMemGetInfo(&memfree,&memtotal); + if(dev==0) *mem_GPU_global=memfree; + if(memfree= 0 && z= 0 && y= 0 && x= cols || y >= rows || z >= depth ) - return; - - - float df[3] ={0.f,0.f,0.f}; - float dfi[3]={0.f,0.f,0.f}; // dfi== \partial f_{i+1,j,k} - float dfj[3]={0.f,0.f,0.f}; - float dfk[3]={0.f,0.f,0.f}; - gradient(f,df ,z ,y ,x , depth,rows,cols); - gradient(f,dfi ,z ,y ,x+1, depth,rows,cols); - gradient(f,dfj ,z ,y+1,x , depth,rows,cols); - gradient(f,dfk ,z+1,y ,x , depth,rows,cols); - float eps=0.00000001; //% avoid division by zero - - dftv[idx]=(df[0]+df[1]+df[2])/(sqrt(df[0] *df[0] +df[1] *df[1] +df[2] *df[2])+eps) - -dfi[2]/(sqrt(dfi[0]*dfi[0]+dfi[1]*dfi[1]+dfi[2]*dfi[2]) +eps) // I wish I coudl precompute this, but if I do then Id need to recompute the gradient. - -dfj[1]/(sqrt(dfj[0]*dfj[0]+dfj[1]*dfj[1]+dfj[2]*dfj[2]) +eps) - -dfk[0]/(sqrt(dfk[0]*dfk[0]+dfk[1]*dfk[1]+dfk[2]*dfk[2]) +eps); - return; - - } - - __device__ void warpReduce(volatile float *sdata, size_t tid) { - sdata[tid] += sdata[tid + 32]; - sdata[tid] += sdata[tid + 16]; - sdata[tid] += sdata[tid + 8]; - sdata[tid] += sdata[tid + 4]; - sdata[tid] += sdata[tid + 2]; - sdata[tid] += sdata[tid + 1]; - } - - __global__ void reduceNorm2(float *g_idata, float *g_odata, size_t n){ - extern __shared__ volatile float sdata[]; - //http://stackoverflow.com/a/35133396/1485872 - size_t tid = threadIdx.x; - size_t i = blockIdx.x*blockDim.x + tid; - size_t gridSize = blockDim.x*gridDim.x; - float mySum = 0; - float value=0; - while (i < n) { - value=g_idata[i]; //avoid reading twice - mySum += value*value; - i += gridSize; - } - sdata[tid] = mySum; - __syncthreads(); - - if (tid < 512) - sdata[tid] += sdata[tid + 512]; - __syncthreads(); - if (tid < 256) - sdata[tid] += sdata[tid + 256]; - __syncthreads(); - - if (tid < 128) - sdata[tid] += sdata[tid + 128]; - __syncthreads(); - - if (tid < 64) - sdata[tid] += sdata[tid + 64]; - __syncthreads(); - - -#if (__CUDART_VERSION >= 9000) - if ( tid < 32 ) - { - mySum = sdata[tid] + sdata[tid + 32]; - for (int offset = warpSize/2; offset > 0; offset /= 2) { - mySum += __shfl_down_sync(0xFFFFFFFF, mySum, offset,32); - } - } -#else - if (tid < 32) { - warpReduce(sdata, tid); - mySum = sdata[0]; - } -#endif - if (tid == 0) g_odata[blockIdx.x] = mySum; - } - - __global__ void reduceSum(float *g_idata, float *g_odata, size_t n){ - extern __shared__ volatile float sdata[]; - //http://stackoverflow.com/a/35133396/1485872 - size_t tid = threadIdx.x; - size_t i = blockIdx.x*blockDim.x + tid; - size_t gridSize = blockDim.x*gridDim.x; - float mySum = 0; - // float value=0; - while (i < n) { - mySum += g_idata[i]; - i += gridSize; - } - sdata[tid] = mySum; - __syncthreads(); - - if (tid < 512) - sdata[tid] += sdata[tid + 512]; - __syncthreads(); - if (tid < 256) - sdata[tid] += sdata[tid + 256]; - __syncthreads(); - - if (tid < 128) - sdata[tid] += sdata[tid + 128]; - __syncthreads(); - - if (tid < 64) - sdata[tid] += sdata[tid + 64]; - __syncthreads(); - - -#if (__CUDART_VERSION >= 9000) - if ( tid < 32 ) - { - mySum = sdata[tid] + sdata[tid + 32]; - for (int offset = warpSize/2; offset > 0; offset /= 2) { - mySum += __shfl_down_sync(0xFFFFFFFF, mySum, offset,32); - } - } -#else - if (tid < 32) { - warpReduce(sdata, tid); - mySum = sdata[0]; - } -#endif - if (tid == 0) g_odata[blockIdx.x] = mySum; - } - - - - -// main function - void pocs_tv(float* img,float* dst,float alpha,const long* image_size, int maxIter, const GpuIds& gpuids){ - - - - - // Prepare for MultiGPU - int deviceCount = gpuids.GetLength(); - cudaCheckErrors("Device query fail"); - if (deviceCount == 0) { - mexErrMsgIdAndTxt("GD_TV:GPU","There are no available device(s) that support CUDA\n"); - } - // - // CODE assumes - // 1.-All available devices are usable by this code - // 2.-All available devices are equal, they are the same machine (warning thrown) - // Check the available devices, and if they are the same - if (!gpuids.AreEqualDevices()) { - mexWarnMsgIdAndTxt("minimizeTV:GD_TV:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); - } - - int dev; - - // We don't know if the devices are being used. lets check that. and only use the amount of memory we need. - - size_t mem_GPU_global; - checkFreeMemory(gpuids, &mem_GPU_global); - - - - // %5 of free memory should be enough, we have almost no variables in these kernels - size_t total_pixels = image_size[0] * image_size[1] * image_size[2] ; - size_t mem_slice_image = sizeof(float)* image_size[0] * image_size[1] ; - size_t mem_size_image = sizeof(float)* total_pixels; - size_t mem_auxiliary = sizeof(float)* (total_pixels + MAXTHREADS - 1) / MAXTHREADS; - - // Decide how are we handling the distribution of computation - size_t mem_img_each_GPU; - - unsigned int buffer_length=2; - //Does everything fit in the GPU? - unsigned int slices_per_split; - - // if it is a thin problem (no need to split), just use one GPU - if (image_size[2]<4){deviceCount=1;} - - unsigned int splits=1; // if the number does not fit in an uint, you have more serious trouble than this. - if(mem_GPU_global> 3*mem_size_image+3*(deviceCount-1)*mem_slice_image*buffer_length+mem_auxiliary){ - // We only need to split if we have extra GPUs - slices_per_split=(image_size[2]+deviceCount-1)/deviceCount; - mem_img_each_GPU=mem_slice_image*((slices_per_split+buffer_length*2)); - }else{ - // As mem_auxiliary is not expected to be a large value (for a 2000^3 image is around 28Mbytes), lets for now assume we need it all - size_t mem_free=mem_GPU_global-mem_auxiliary; - - splits=(unsigned int)(ceil(((float)(3*mem_size_image)/(float)(deviceCount))/mem_free)); - // Now, there is an overhead here, as each splits should have 2 slices more, to account for overlap of images. - // lets make sure these 2 slices fit, if they do not, add 1 to splits. - slices_per_split=(image_size[2]+deviceCount*splits-1)/(deviceCount*splits); - mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); - - // if the new stuff does not fit in the GPU, it means we are in the edge case where adding that extra slice will overflow memory - if (mem_GPU_global< 3*mem_img_each_GPU+mem_auxiliary){ - // one more split should do the job, as its an edge case. - splits++; - //recompute for later - slices_per_split=(image_size[2]+deviceCount*splits-1)/(deviceCount*splits); // amount of slices that fit on a GPU. Later we add 2 to these, as we need them for overlap - mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); - } - - - // How many EXTRA buffer slices should be able to fit in here??!?! - // Only do it if there are splits needed. - if(splits>1){ - mem_free=mem_GPU_global-(3*mem_img_each_GPU+mem_auxiliary); - unsigned int extra_buff=(mem_free/mem_slice_image); - buffer_length=(extra_buff/2)/3; // we need double whatever this results in, rounded down. - buffer_length=max(buffer_length,2);// minimum 2 - buffer_length=min(MAX_BUFFER,buffer_length); - - mem_img_each_GPU=mem_slice_image*(slices_per_split+buffer_length*2); - - }else{ - buffer_length=2; - } - - // Assert - if (mem_GPU_global< 3*mem_img_each_GPU+mem_auxiliary){ - mexErrMsgIdAndTxt("GD_TV:GPU","Assertion Failed. Logic behind splitting flawed! Please tell: ander.biguri@gmail.com\n"); - } - } - - - // Assert - - if ((slices_per_split+buffer_length*2)*image_size[0]*image_size[1]* sizeof(float)!= mem_img_each_GPU){ - mexErrMsgIdAndTxt("GD_TV:GPU","Assertion Failed. Memory needed calculation broken! Please tell: ander.biguri@gmail.com\n"); - } - - - - - - - float** d_image= (float**)malloc(deviceCount*sizeof(float*)); - float** d_dimgTV= (float**)malloc(deviceCount*sizeof(float*)); - float** d_norm2aux= (float**)malloc(deviceCount*sizeof(float*)); - float** d_norm2= (float**)malloc(deviceCount*sizeof(float*)); - - // allocate memory in each GPU - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - - cudaMalloc((void**)&d_image[dev] , mem_img_each_GPU); - cudaMemset( d_image[dev],0 , mem_img_each_GPU); - cudaMalloc((void**)&d_dimgTV[dev] , mem_img_each_GPU); - cudaMemset( d_dimgTV[dev],0 , mem_img_each_GPU); - cudaMalloc((void**)&d_norm2[dev] , slices_per_split*mem_slice_image); - cudaMemset( d_norm2[dev],0 , slices_per_split*mem_slice_image); - cudaMalloc((void**)&d_norm2aux[dev] , mem_auxiliary); - cudaMemset( d_norm2aux[dev],0 , mem_auxiliary); - cudaCheckErrors("Malloc error"); - - - } - unsigned long long buffer_pixels=buffer_length*image_size[0]*image_size[1]; - float* buffer; - if(splits>1){ - mexWarnMsgIdAndTxt("minimizeTV:GD_TV:Image_split","Your image can not be fully split between the available GPUs. The computation of minTV will be significantly slowed due to the image size.\nApproximated mathematics turned on for computational speed."); - }else{ - cudaMallocHost((void**)&buffer,buffer_length*image_size[0]*image_size[1]*sizeof(float)); - } - - - - // Lets try to make the host memory pinned: - // We laredy queried the GPU and assuemd they are the same, thus should have the same attributes. - int isHostRegisterSupported = 0; -#if CUDART_VERSION >= 9020 - cudaDeviceGetAttribute(&isHostRegisterSupported,cudaDevAttrHostRegisterSupported,gpuids[0]); -#endif - // splits>2 is completely empirical observation - if (isHostRegisterSupported & splits>2){ - cudaHostRegister(img ,image_size[2]*image_size[1]*image_size[0]*sizeof(float),cudaHostRegisterPortable); - cudaHostRegister(dst ,image_size[2]*image_size[1]*image_size[0]*sizeof(float),cudaHostRegisterPortable); - } - cudaCheckErrors("Error pinning memory"); - - - - // Create streams - int nStream_device=2; - int nStreams=deviceCount*nStream_device; - cudaStream_t* stream=(cudaStream_t*)malloc(nStreams*sizeof(cudaStream_t)); - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - for (int i = 0; i < nStream_device; ++i){ - cudaStreamCreate(&stream[i+dev*nStream_device]); - } - } - cudaCheckErrors("Stream creation fail"); - - - // For the reduction - - double totalsum_prev; - double totalsum; - float sum_curr_spl; - float * sumnorm2; - cudaMallocHost((void**)&sumnorm2,deviceCount*sizeof(float)); - - unsigned int curr_slices; - unsigned long long curr_pixels; - size_t linear_idx_start; - unsigned long long* offset_device=(unsigned long long*)malloc(deviceCount*sizeof(unsigned long long)); - unsigned long long* offset_host =(unsigned long long*)malloc(deviceCount*sizeof(unsigned long long)); - unsigned long long* bytes_device =(unsigned long long*)malloc(deviceCount*sizeof(unsigned long long)); - bool is_first_chunk; - bool is_last_chunk; - for(unsigned int i=0;i1){ - totalsum_prev=0; - } - for(unsigned int sp=0;sp1 & i>0){ - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaMemcpyAsync(d_image[dev]+offset_device[dev], dst+offset_host[dev] , bytes_device[dev]*sizeof(float), cudaMemcpyHostToDevice,stream[dev*nStream_device+1]); - - - } - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - } - } - cudaCheckErrors("Memcpy failure on multi split"); - - for(unsigned int ib=0; (ib<(buffer_length-1)) && ((i+ib)>>(d_image[dev],d_dimgTV[dev],(long)(curr_slices+buffer_length*2-1), image_size[1],image_size[0]); - - } - - - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - curr_slices=((sp*deviceCount+dev+1)*slices_per_split> >(d_norm2[dev], d_norm2aux[dev], total_pixels); - - } - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - curr_slices=((sp*deviceCount+dev+1)*slices_per_split 1) { - reduceSum << <1, dimblockRed, MAXTHREADS*sizeof(float),stream[dev*nStream_device] >> >(d_norm2aux[dev], d_norm2[dev], dimgridRed); - cudaStreamSynchronize(stream[dev*nStream_device]); - cudaMemcpyAsync(&sumnorm2[dev], d_norm2[dev], sizeof(float), cudaMemcpyDeviceToHost,stream[dev*nStream_device+1]); - } - else { - cudaStreamSynchronize(stream[dev*nStream_device]); - cudaMemcpyAsync(&sumnorm2[dev], d_norm2aux[dev], sizeof(float), cudaMemcpyDeviceToHost,stream[dev*nStream_device+1]); - } - } - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - } - cudaCheckErrors("Reduction error"); - - - // Accumulate the norm accross devices - sum_curr_spl=0; - // this is CPU code - for (dev = 0; dev < deviceCount; dev++){ - sum_curr_spl+=sumnorm2[dev]; - } - sum_curr_spl+=0.0000001f; // avoid division by zero - - // If we have more than one splits, lets use the result from prior calls - if(i>0 && splits>1){ - // this is already stored: - //totalsum=totalsum_prev; - }else{ - totalsum=sum_curr_spl; - } - - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - curr_slices=((sp*deviceCount+dev+1)*slices_per_split>>(d_dimgTV[dev]+buffer_pixels,(float)sqrt(totalsum),total_pixels); - //MULTIPLY HYPERPARAMETER - multiplyArrayScalar<<<60,MAXTHREADS,0,stream[dev*nStream_device]>>>(d_dimgTV[dev]+buffer_pixels,alpha, total_pixels); - } - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - } - cudaCheckErrors("Scalar operations error"); - - //SUBSTRACT GRADIENT - ////////////////////////////////////////////// - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - curr_slices=((sp*deviceCount+dev+1)*slices_per_split>>(d_image[dev]+buffer_pixels,d_dimgTV[dev]+buffer_pixels, total_pixels); - } - } - - // Synchronize mathematics, make sure bounding pixels are correct - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - } - - if(splits==1){ - for(dev=0; dev0){ - cudaSetDevice(gpuids[dev-1]); - cudaMemcpyAsync(buffer, d_image[dev-1]+total_pixels+buffer_pixels, buffer_pixels*sizeof(float), cudaMemcpyDeviceToHost); - cudaSetDevice(gpuids[dev]); - cudaMemcpyAsync(d_image[dev],buffer, buffer_pixels*sizeof(float), cudaMemcpyHostToDevice); - } - } - }else{ - - // We need to take it out :( - for(dev=0; dev2){ - cudaHostUnregister(img); - cudaHostUnregister(dst); - } - for (int i = 0; i < nStreams; ++i) - cudaStreamDestroy(stream[i]) ; - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - } - cudaCheckErrors("Memory free"); - // Do NOT cudaDeviceReset() here: it destroys the PRIMARY CUDA - // CONTEXT of the whole host process, not just this function's - // resources - every allocation, module and stream the caller holds - // through CuPy, PyTorch (see the d25 bindings) or any other CUDA - // library dies with it. TIGRE's own subsequent kernels survive only - // because the runtime silently re-creates the context, so the - // symptom appears in the NEXT library to touch the GPU (e.g. - // cudaErrorInvalidValue / CUDA_ERROR_INVALID_HANDLE inside CuPy - // deallocations) rather than in TIGRE itself. All buffers are - // already freed explicitly above; GD_AwTV.cu has this reset - // commented out for the same reason. - // cudaDeviceReset(); - } - -void checkFreeMemory(const GpuIds& gpuids,size_t *mem_GPU_global){ - size_t memfree; - size_t memtotal; - int deviceCount = gpuids.GetLength(); - for (int dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaMemGetInfo(&memfree,&memtotal); - if(dev==0) *mem_GPU_global=memfree; - if(memfree= 0 && z= 0 && y= 0 && x= cols || y >= rows || z >= depth ) + return; + + + float df[3] ={0.f,0.f,0.f}; + float dfi[3]={0.f,0.f,0.f}; // dfi== \partial f_{i+1,j,k} + float dfj[3]={0.f,0.f,0.f}; + float dfk[3]={0.f,0.f,0.f}; + gradient(f,df ,z ,y ,x , depth,rows,cols); + gradient(f,dfi ,z ,y ,x+1, depth,rows,cols); + gradient(f,dfj ,z ,y+1,x , depth,rows,cols); + gradient(f,dfk ,z+1,y ,x , depth,rows,cols); + float eps=0.00000001; //% avoid division by zero + + dftv[idx]=(df[0]+df[1]+df[2])/(sqrt(df[0] *df[0] +df[1] *df[1] +df[2] *df[2])+eps) + -dfi[2]/(sqrt(dfi[0]*dfi[0]+dfi[1]*dfi[1]+dfi[2]*dfi[2]) +eps) // I wish I coudl precompute this, but if I do then Id need to recompute the gradient. + -dfj[1]/(sqrt(dfj[0]*dfj[0]+dfj[1]*dfj[1]+dfj[2]*dfj[2]) +eps) + -dfk[0]/(sqrt(dfk[0]*dfk[0]+dfk[1]*dfk[1]+dfk[2]*dfk[2]) +eps); + return; + + } + + __device__ void warpReduce(volatile float *sdata, size_t tid) { + sdata[tid] += sdata[tid + 32]; + sdata[tid] += sdata[tid + 16]; + sdata[tid] += sdata[tid + 8]; + sdata[tid] += sdata[tid + 4]; + sdata[tid] += sdata[tid + 2]; + sdata[tid] += sdata[tid + 1]; + } + + __global__ void reduceNorm2(float *g_idata, float *g_odata, size_t n){ + extern __shared__ volatile float sdata[]; + //http://stackoverflow.com/a/35133396/1485872 + size_t tid = threadIdx.x; + size_t i = blockIdx.x*blockDim.x + tid; + size_t gridSize = blockDim.x*gridDim.x; + float mySum = 0; + float value=0; + while (i < n) { + value=g_idata[i]; //avoid reading twice + mySum += value*value; + i += gridSize; + } + sdata[tid] = mySum; + __syncthreads(); + + if (tid < 512) + sdata[tid] += sdata[tid + 512]; + __syncthreads(); + if (tid < 256) + sdata[tid] += sdata[tid + 256]; + __syncthreads(); + + if (tid < 128) + sdata[tid] += sdata[tid + 128]; + __syncthreads(); + + if (tid < 64) + sdata[tid] += sdata[tid + 64]; + __syncthreads(); + + +#if (__CUDART_VERSION >= 9000) + if ( tid < 32 ) + { + mySum = sdata[tid] + sdata[tid + 32]; + for (int offset = warpSize/2; offset > 0; offset /= 2) { + mySum += __shfl_down_sync(0xFFFFFFFF, mySum, offset,32); + } + } +#else + if (tid < 32) { + warpReduce(sdata, tid); + mySum = sdata[0]; + } +#endif + if (tid == 0) g_odata[blockIdx.x] = mySum; + } + + __global__ void reduceSum(float *g_idata, float *g_odata, size_t n){ + extern __shared__ volatile float sdata[]; + //http://stackoverflow.com/a/35133396/1485872 + size_t tid = threadIdx.x; + size_t i = blockIdx.x*blockDim.x + tid; + size_t gridSize = blockDim.x*gridDim.x; + float mySum = 0; + // float value=0; + while (i < n) { + mySum += g_idata[i]; + i += gridSize; + } + sdata[tid] = mySum; + __syncthreads(); + + if (tid < 512) + sdata[tid] += sdata[tid + 512]; + __syncthreads(); + if (tid < 256) + sdata[tid] += sdata[tid + 256]; + __syncthreads(); + + if (tid < 128) + sdata[tid] += sdata[tid + 128]; + __syncthreads(); + + if (tid < 64) + sdata[tid] += sdata[tid + 64]; + __syncthreads(); + + +#if (__CUDART_VERSION >= 9000) + if ( tid < 32 ) + { + mySum = sdata[tid] + sdata[tid + 32]; + for (int offset = warpSize/2; offset > 0; offset /= 2) { + mySum += __shfl_down_sync(0xFFFFFFFF, mySum, offset,32); + } + } +#else + if (tid < 32) { + warpReduce(sdata, tid); + mySum = sdata[0]; + } +#endif + if (tid == 0) g_odata[blockIdx.x] = mySum; + } + + + + +// main function + void pocs_tv(float* img,float* dst,float alpha,const long* image_size, int maxIter, const GpuIds& gpuids){ + + + + + // Prepare for MultiGPU + int deviceCount = gpuids.GetLength(); + hipCheckErrors("Device query fail"); + if (deviceCount == 0) { + mexErrMsgIdAndTxt("GD_TV:GPU","There are no available device(s) that support CUDA\n"); + } + // + // CODE assumes + // 1.-All available devices are usable by this code + // 2.-All available devices are equal, they are the same machine (warning thrown) + // Check the available devices, and if they are the same + if (!gpuids.AreEqualDevices()) { + mexWarnMsgIdAndTxt("minimizeTV:GD_TV:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); + } + + int dev; + + // We don't know if the devices are being used. lets check that. and only use the amount of memory we need. + + size_t mem_GPU_global; + checkFreeMemory(gpuids, &mem_GPU_global); + + + + // %5 of free memory should be enough, we have almost no variables in these kernels + size_t total_pixels = image_size[0] * image_size[1] * image_size[2] ; + size_t mem_slice_image = sizeof(float)* image_size[0] * image_size[1] ; + size_t mem_size_image = sizeof(float)* total_pixels; + size_t mem_auxiliary = sizeof(float)* (total_pixels + MAXTHREADS - 1) / MAXTHREADS; + + // Decide how are we handling the distribution of computation + size_t mem_img_each_GPU; + + unsigned int buffer_length=2; + //Does everything fit in the GPU? + unsigned int slices_per_split; + + // if it is a thin problem (no need to split), just use one GPU + if (image_size[2]<4){deviceCount=1;} + + unsigned int splits=1; // if the number does not fit in an uint, you have more serious trouble than this. + if(mem_GPU_global> 3*mem_size_image+3*(deviceCount-1)*mem_slice_image*buffer_length+mem_auxiliary){ + // We only need to split if we have extra GPUs + slices_per_split=(image_size[2]+deviceCount-1)/deviceCount; + mem_img_each_GPU=mem_slice_image*((slices_per_split+buffer_length*2)); + }else{ + // As mem_auxiliary is not expected to be a large value (for a 2000^3 image is around 28Mbytes), lets for now assume we need it all + size_t mem_free=mem_GPU_global-mem_auxiliary; + + splits=(unsigned int)(ceil(((float)(3*mem_size_image)/(float)(deviceCount))/mem_free)); + // Now, there is an overhead here, as each splits should have 2 slices more, to account for overlap of images. + // lets make sure these 2 slices fit, if they do not, add 1 to splits. + slices_per_split=(image_size[2]+deviceCount*splits-1)/(deviceCount*splits); + mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); + + // if the new stuff does not fit in the GPU, it means we are in the edge case where adding that extra slice will overflow memory + if (mem_GPU_global< 3*mem_img_each_GPU+mem_auxiliary){ + // one more split should do the job, as its an edge case. + splits++; + //recompute for later + slices_per_split=(image_size[2]+deviceCount*splits-1)/(deviceCount*splits); // amount of slices that fit on a GPU. Later we add 2 to these, as we need them for overlap + mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); + } + + + // How many EXTRA buffer slices should be able to fit in here??!?! + // Only do it if there are splits needed. + if(splits>1){ + mem_free=mem_GPU_global-(3*mem_img_each_GPU+mem_auxiliary); + unsigned int extra_buff=(mem_free/mem_slice_image); + buffer_length=(extra_buff/2)/3; // we need double whatever this results in, rounded down. + buffer_length=max(buffer_length,2);// minimum 2 + buffer_length=min(MAX_BUFFER,buffer_length); + + mem_img_each_GPU=mem_slice_image*(slices_per_split+buffer_length*2); + + }else{ + buffer_length=2; + } + + // Assert + if (mem_GPU_global< 3*mem_img_each_GPU+mem_auxiliary){ + mexErrMsgIdAndTxt("GD_TV:GPU","Assertion Failed. Logic behind splitting flawed! Please tell: ander.biguri@gmail.com\n"); + } + } + + + // Assert + + if ((slices_per_split+buffer_length*2)*image_size[0]*image_size[1]* sizeof(float)!= mem_img_each_GPU){ + mexErrMsgIdAndTxt("GD_TV:GPU","Assertion Failed. Memory needed calculation broken! Please tell: ander.biguri@gmail.com\n"); + } + + + + + + + float** d_image= (float**)malloc(deviceCount*sizeof(float*)); + float** d_dimgTV= (float**)malloc(deviceCount*sizeof(float*)); + float** d_norm2aux= (float**)malloc(deviceCount*sizeof(float*)); + float** d_norm2= (float**)malloc(deviceCount*sizeof(float*)); + + // allocate memory in each GPU + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + + hipMalloc((void**)&d_image[dev] , mem_img_each_GPU); + hipMemset( d_image[dev],0 , mem_img_each_GPU); + hipMalloc((void**)&d_dimgTV[dev] , mem_img_each_GPU); + hipMemset( d_dimgTV[dev],0 , mem_img_each_GPU); + hipMalloc((void**)&d_norm2[dev] , slices_per_split*mem_slice_image); + hipMemset( d_norm2[dev],0 , slices_per_split*mem_slice_image); + hipMalloc((void**)&d_norm2aux[dev] , mem_auxiliary); + hipMemset( d_norm2aux[dev],0 , mem_auxiliary); + hipCheckErrors("Malloc error"); + + + } + unsigned long long buffer_pixels=buffer_length*image_size[0]*image_size[1]; + float* buffer; + if(splits>1){ + mexWarnMsgIdAndTxt("minimizeTV:GD_TV:Image_split","Your image can not be fully split between the available GPUs. The computation of minTV will be significantly slowed due to the image size.\nApproximated mathematics turned on for computational speed."); + }else{ + hipMallocHost((void**)&buffer,buffer_length*image_size[0]*image_size[1]*sizeof(float)); + } + + + + // Lets try to make the host memory pinned: + // We laredy queried the GPU and assuemd they are the same, thus should have the same attributes. + int isHostRegisterSupported = 0; +#if CUDART_VERSION >= 9020 + hipDeviceGetAttribute(&isHostRegisterSupported,hipDevAttrHostRegisterSupported,gpuids[0]); +#endif + // splits>2 is completely empirical observation + if (isHostRegisterSupported & splits>2){ + hipHostRegister(img ,image_size[2]*image_size[1]*image_size[0]*sizeof(float),hipHostRegisterPortable); + hipHostRegister(dst ,image_size[2]*image_size[1]*image_size[0]*sizeof(float),hipHostRegisterPortable); + } + hipCheckErrors("Error pinning memory"); + + + + // Create streams + int nStream_device=2; + int nStreams=deviceCount*nStream_device; + hipStream_t* stream=(hipStream_t*)malloc(nStreams*sizeof(hipStream_t)); + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + for (int i = 0; i < nStream_device; ++i){ + hipStreamCreate(&stream[i+dev*nStream_device]); + } + } + hipCheckErrors("Stream creation fail"); + + + // For the reduction + + double totalsum_prev; + double totalsum; + float sum_curr_spl; + float * sumnorm2; + hipMallocHost((void**)&sumnorm2,deviceCount*sizeof(float)); + + unsigned int curr_slices; + unsigned long long curr_pixels; + size_t linear_idx_start; + unsigned long long* offset_device=(unsigned long long*)malloc(deviceCount*sizeof(unsigned long long)); + unsigned long long* offset_host =(unsigned long long*)malloc(deviceCount*sizeof(unsigned long long)); + unsigned long long* bytes_device =(unsigned long long*)malloc(deviceCount*sizeof(unsigned long long)); + bool is_first_chunk; + bool is_last_chunk; + for(unsigned int i=0;i1){ + totalsum_prev=0; + } + for(unsigned int sp=0;sp1 & i>0){ + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipMemcpyAsync(d_image[dev]+offset_device[dev], dst+offset_host[dev] , bytes_device[dev]*sizeof(float), hipMemcpyHostToDevice,stream[dev*nStream_device+1]); + + + } + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + } + } + hipCheckErrors("Memcpy failure on multi split"); + + for(unsigned int ib=0; (ib<(buffer_length-1)) && ((i+ib)>>(d_image[dev],d_dimgTV[dev],(long)(curr_slices+buffer_length*2-1), image_size[1],image_size[0]); + + } + + + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + curr_slices=((sp*deviceCount+dev+1)*slices_per_split> >(d_norm2[dev], d_norm2aux[dev], total_pixels); + + } + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + curr_slices=((sp*deviceCount+dev+1)*slices_per_split 1) { + reduceSum << <1, dimblockRed, MAXTHREADS*sizeof(float),stream[dev*nStream_device] >> >(d_norm2aux[dev], d_norm2[dev], dimgridRed); + hipStreamSynchronize(stream[dev*nStream_device]); + hipMemcpyAsync(&sumnorm2[dev], d_norm2[dev], sizeof(float), hipMemcpyDeviceToHost,stream[dev*nStream_device+1]); + } + else { + hipStreamSynchronize(stream[dev*nStream_device]); + hipMemcpyAsync(&sumnorm2[dev], d_norm2aux[dev], sizeof(float), hipMemcpyDeviceToHost,stream[dev*nStream_device+1]); + } + } + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + } + hipCheckErrors("Reduction error"); + + + // Accumulate the norm accross devices + sum_curr_spl=0; + // this is CPU code + for (dev = 0; dev < deviceCount; dev++){ + sum_curr_spl+=sumnorm2[dev]; + } + sum_curr_spl+=0.0000001f; // avoid division by zero + + // If we have more than one splits, lets use the result from prior calls + if(i>0 && splits>1){ + // this is already stored: + //totalsum=totalsum_prev; + }else{ + totalsum=sum_curr_spl; + } + + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + curr_slices=((sp*deviceCount+dev+1)*slices_per_split>>(d_dimgTV[dev]+buffer_pixels,(float)sqrt(totalsum),total_pixels); + //MULTIPLY HYPERPARAMETER + multiplyArrayScalar<<<60,MAXTHREADS,0,stream[dev*nStream_device]>>>(d_dimgTV[dev]+buffer_pixels,alpha, total_pixels); + } + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + } + hipCheckErrors("Scalar operations error"); + + //SUBSTRACT GRADIENT + ////////////////////////////////////////////// + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + curr_slices=((sp*deviceCount+dev+1)*slices_per_split>>(d_image[dev]+buffer_pixels,d_dimgTV[dev]+buffer_pixels, total_pixels); + } + } + + // Synchronize mathematics, make sure bounding pixels are correct + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + } + + if(splits==1){ + for(dev=0; dev0){ + hipSetDevice(gpuids[dev-1]); + hipMemcpyAsync(buffer, d_image[dev-1]+total_pixels+buffer_pixels, buffer_pixels*sizeof(float), hipMemcpyDeviceToHost); + hipSetDevice(gpuids[dev]); + hipMemcpyAsync(d_image[dev],buffer, buffer_pixels*sizeof(float), hipMemcpyHostToDevice); + } + } + }else{ + + // We need to take it out :( + for(dev=0; dev2){ + hipHostUnregister(img); + hipHostUnregister(dst); + } + for (int i = 0; i < nStreams; ++i) + hipStreamDestroy(stream[i]) ; + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + } + hipCheckErrors("Memory free"); + // Do NOT hipDeviceReset() here: it destroys the PRIMARY CUDA + // CONTEXT of the whole host process, not just this function's + // resources - every allocation, module and stream the caller holds + // through CuPy, PyTorch (see the d25 bindings) or any other CUDA + // library dies with it. TIGRE's own subsequent kernels survive only + // because the runtime silently re-creates the context, so the + // symptom appears in the NEXT library to touch the GPU (e.g. + // hipErrorInvalidValue / CUDA_ERROR_INVALID_HANDLE inside CuPy + // deallocations) rather than in TIGRE itself. All buffers are + // already freed explicitly above; GD_AwTV.cu has this reset + // commented out for the same reason. + // hipDeviceReset(); + } + +void checkFreeMemory(const GpuIds& gpuids,size_t *mem_GPU_global){ + size_t memfree; + size_t memtotal; + int deviceCount = gpuids.GetLength(); + for (int dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipMemGetInfo(&memfree,&memtotal); + if(dev==0) *mem_GPU_global=memfree; + if(memfree -#include -#include - -GpuIds::~GpuIds() { - free(m_piDeviceIds); m_piDeviceIds = nullptr; - m_iCount = 0; -} -GpuIds::GpuIds() : m_piDeviceIds (nullptr), m_iCount(0) { - -} -void GpuIds::SetIds(int iCount, int* piDeviceIds) { - if (iCount > 0 && piDeviceIds != 0) { - if (m_piDeviceIds) { - free(m_piDeviceIds); m_piDeviceIds = nullptr; - m_iCount = 0; - } - m_piDeviceIds = (int*)malloc(iCount * sizeof(int)); - if (m_piDeviceIds) { - for (int iI = 0; iI < iCount; ++iI) { - m_piDeviceIds[iI] = piDeviceIds[iI]; - } - m_iCount = iCount; - } - } -} - -int GpuIds::GetLength() const { - return m_iCount; -} -int& GpuIds::operator[](int iIndex){ - return m_piDeviceIds[iIndex]; -} -int GpuIds::operator[](int iIndex) const { - return m_piDeviceIds[iIndex]; -} - -void GpuIds::SetAllGpus(int iTotalDeviceCount) { - // Set all GPUs for compatibility - // Makeup valid GpuIds. - int* aiIds = nullptr; - if (iTotalDeviceCount == 0) { - (int*)malloc(iTotalDeviceCount*sizeof(int)); - for (int iI = 0; iI < iTotalDeviceCount; ++iI) { - aiIds[iI] = iI; - } - } - SetIds(iTotalDeviceCount, aiIds); - free(aiIds); aiIds = 0; -} - -bool GpuIds::AreEqualDevices() const { - int deviceCount = this->GetLength(); - const int devicenamelength = 256; // The length 256 is fixed by spec of cudaDeviceProp::name - char devicename[devicenamelength]; - cudaDeviceProp deviceProp; - for (int dev = 0; dev < deviceCount; dev++) { - // cudaSetDevice(m_piDeviceIds[dev]); - cudaGetDeviceProperties(&deviceProp, m_piDeviceIds[dev]); - if (dev>0) { - if (strcmp(devicename, deviceProp.name) != 0) { - return false; - } - } - memset(devicename, 0, devicenamelength); - strcpy(devicename, deviceProp.name); - } - return true; -} +#include "GpuIds.hpp" +#include +#include +#include + +GpuIds::~GpuIds() { + free(m_piDeviceIds); m_piDeviceIds = nullptr; + m_iCount = 0; +} +GpuIds::GpuIds() : m_piDeviceIds (nullptr), m_iCount(0) { + +} +void GpuIds::SetIds(int iCount, int* piDeviceIds) { + if (iCount > 0 && piDeviceIds != 0) { + if (m_piDeviceIds) { + free(m_piDeviceIds); m_piDeviceIds = nullptr; + m_iCount = 0; + } + m_piDeviceIds = (int*)malloc(iCount * sizeof(int)); + if (m_piDeviceIds) { + for (int iI = 0; iI < iCount; ++iI) { + m_piDeviceIds[iI] = piDeviceIds[iI]; + } + m_iCount = iCount; + } + } +} + +int GpuIds::GetLength() const { + return m_iCount; +} +int& GpuIds::operator[](int iIndex){ + return m_piDeviceIds[iIndex]; +} +int GpuIds::operator[](int iIndex) const { + return m_piDeviceIds[iIndex]; +} + +void GpuIds::SetAllGpus(int iTotalDeviceCount) { + // Set all GPUs for compatibility + // Makeup valid GpuIds. + int* aiIds = nullptr; + if (iTotalDeviceCount == 0) { + (int*)malloc(iTotalDeviceCount*sizeof(int)); + for (int iI = 0; iI < iTotalDeviceCount; ++iI) { + aiIds[iI] = iI; + } + } + SetIds(iTotalDeviceCount, aiIds); + free(aiIds); aiIds = 0; +} + +bool GpuIds::AreEqualDevices() const { + int deviceCount = this->GetLength(); + const int devicenamelength = 256; // The length 256 is fixed by spec of hipDeviceProp::name + char devicename[devicenamelength]; + hipDeviceProp deviceProp; + for (int dev = 0; dev < deviceCount; dev++) { + // hipSetDevice(m_piDeviceIds[dev]); + hipGetDeviceProperties(&deviceProp, m_piDeviceIds[dev]); + if (dev>0) { + if (strcmp(devicename, deviceProp.name) != 0) { + return false; + } + } + memset(devicename, 0, devicenamelength); + strcpy(devicename, deviceProp.name); + } + return true; +} diff --git a/Common/CUDA/GpuIds.hpp b/Common/CUDA/GpuIds.hpp index e0223f86..933aff5c 100644 --- a/Common/CUDA/GpuIds.hpp +++ b/Common/CUDA/GpuIds.hpp @@ -1,17 +1,17 @@ - -#ifndef GPUIDS_H -#define GPUIDS_H -struct GpuIds { - int* m_piDeviceIds; - int m_iCount; - ~GpuIds(); - GpuIds(); - void SetIds(int iCount, int* piDeviceIds); - int GetLength() const; - void SetAllGpus(int iTotalDeviceCount); - int& operator[](int iIndex); - int operator[](int iIndex) const; - bool AreEqualDevices() const; -}; -#endif - + +#ifndef GPUIDS_H +#define GPUIDS_H +struct GpuIds { + int* m_piDeviceIds; + int m_iCount; + ~GpuIds(); + GpuIds(); + void SetIds(int iCount, int* piDeviceIds); + int GetLength() const; + void SetAllGpus(int iTotalDeviceCount); + int& operator[](int iIndex); + int operator[](int iIndex) const; + bool AreEqualDevices() const; +}; +#endif + diff --git a/Common/CUDA/PICCS.cu b/Common/CUDA/PICCS.cu index 34ecdc58..27735957 100644 --- a/Common/CUDA/PICCS.cu +++ b/Common/CUDA/PICCS.cu @@ -58,12 +58,12 @@ Codes : https://github.com/CERN/TIGRE -#define cudaCheckErrors(msg) \ +#define hipCheckErrors(msg) \ do { \ - cudaError_t __err = cudaGetLastError(); \ - if (__err != cudaSuccess) { \ + hipError_t __err = hipGetLastError(); \ + if (__err != hipSuccess) { \ mexPrintf("ERROR in: %s \n",msg);\ - mexErrMsgIdAndTxt("err",cudaGetErrorString(__err));\ + mexErrMsgIdAndTxt("err",hipGetErrorString(__err));\ } \ } while (0) @@ -263,9 +263,9 @@ do { \ bool isnan_cuda(float* vec, size_t size){ bool*d_nan; bool h_nan; - cudaMalloc((void **)&d_nan, sizeof (bool)); + hipMalloc((void **)&d_nan, sizeof (bool)); isnan_device<<<60,MAXTHREADS>>>(vec,size,d_nan); - cudaMemcpy(&h_nan, d_nan, sizeof(bool), cudaMemcpyDeviceToHost); + hipMemcpy(&h_nan, d_nan, sizeof(bool), hipMemcpyDeviceToHost); return h_nan; } @@ -281,25 +281,25 @@ bool isnan_cuda(float* vec, size_t size){ float *d_image,*d_prior,*d_dpiccsTV, *d_dimgTV,*d_aux_small,*d_aux_image, *d_norm2; // memory for image - cudaMalloc(&d_image, mem_size); - cudaMalloc(&d_prior, mem_size); + hipMalloc(&d_image, mem_size); + hipMalloc(&d_prior, mem_size); - cudaCheckErrors("Malloc Image error"); - cudaMemcpy(d_image, img, mem_size, cudaMemcpyHostToDevice); - cudaMemcpy(d_prior, prior, mem_size, cudaMemcpyHostToDevice); - cudaCheckErrors("Memory Malloc and Memset: SRC"); + hipCheckErrors("Malloc Image error"); + hipMemcpy(d_image, img, mem_size, hipMemcpyHostToDevice); + hipMemcpy(d_prior, prior, mem_size, hipMemcpyHostToDevice); + hipCheckErrors("Memory Malloc and Memset: SRC"); // memory for df - cudaMalloc(&d_dimgTV, mem_size); - cudaMalloc(&d_dpiccsTV, mem_size); - cudaCheckErrors("Memory Malloc and Memset: TV"); - cudaMalloc(&d_norm2, mem_size); - cudaCheckErrors("Memory Malloc and Memset: TV"); - cudaMalloc(&d_aux_image, mem_size); - cudaCheckErrors("Memory Malloc and Memset: TV"); + hipMalloc(&d_dimgTV, mem_size); + hipMalloc(&d_dpiccsTV, mem_size); + hipCheckErrors("Memory Malloc and Memset: TV"); + hipMalloc(&d_norm2, mem_size); + hipCheckErrors("Memory Malloc and Memset: TV"); + hipMalloc(&d_aux_image, mem_size); + hipCheckErrors("Memory Malloc and Memset: TV"); // memory for L2norm auxiliar - cudaMalloc(&d_aux_small, sizeof(float)*(total_pixels + MAXTHREADS - 1) / MAXTHREADS); - cudaCheckErrors("Memory Malloc and Memset: NORMAux"); + hipMalloc(&d_aux_small, sizeof(float)*(total_pixels + MAXTHREADS - 1) / MAXTHREADS); + hipCheckErrors("Memory Malloc and Memset: NORMAux"); @@ -315,88 +315,88 @@ bool isnan_cuda(float* vec, size_t size){ for(unsigned int i=0;i>>(d_image,d_dimgTV,image_size[2], image_size[1],image_size[0]); - cudaDeviceSynchronize(); - cudaCheckErrors("Gradient"); + hipDeviceSynchronize(); + hipCheckErrors("Gradient"); // mexPrintf("Gradient is nan: %s\n",isnan_cuda(d_dimgTV,total_pixels) ? "true" : "false"); multiplyArrayScalar<<<60,MAXTHREADS>>>(d_dimgTV,(1-ratio), total_pixels); - cudaDeviceSynchronize(); - cudaCheckErrors("Multiplication error"); + hipDeviceSynchronize(); + hipCheckErrors("Multiplication error"); substractArrays<<<60,MAXTHREADS>>>(d_aux_image,d_prior, total_pixels); - cudaDeviceSynchronize(); - cudaCheckErrors("Substraction error"); + hipDeviceSynchronize(); + hipCheckErrors("Substraction error"); gradientTV<<>>(d_aux_image,d_dpiccsTV,image_size[2], image_size[1],image_size[0]); - cudaDeviceSynchronize(); - cudaCheckErrors("Gradient"); + hipDeviceSynchronize(); + hipCheckErrors("Gradient"); // mexPrintf("Gradient piccs is nan: %s\n",isnan_cuda(d_dimgTV,total_pixels) ? "true" : "false"); multiplyArrayScalar<<<60,MAXTHREADS>>>(d_dpiccsTV,ratio, total_pixels); - cudaDeviceSynchronize(); - cudaCheckErrors("Multiplication error"); + hipDeviceSynchronize(); + hipCheckErrors("Multiplication error"); // mexPrintf("Multiplication is nan: %s\n",isnan_cuda(d_dimgTV,total_pixels) ? "true" : "false"); addArrays<<<60,MAXTHREADS>>>(d_dimgTV,d_dpiccsTV,total_pixels); - cudaDeviceSynchronize(); + hipDeviceSynchronize(); //NOMRALIZE via reduction //mexPrintf("Pre-norm2 is nan: %s\n",isnan_cuda(d_dimgTV,total_pixels) ? "true" : "false"); - cudaMemcpy(d_norm2, d_dimgTV, mem_size, cudaMemcpyDeviceToDevice); - cudaCheckErrors("Copy from gradient call error"); + hipMemcpy(d_norm2, d_dimgTV, mem_size, hipMemcpyDeviceToDevice); + hipCheckErrors("Copy from gradient call error"); reduceNorm2 << > >(d_norm2, d_aux_small, total_pixels); - cudaDeviceSynchronize(); - cudaCheckErrors("reduce1"); + hipDeviceSynchronize(); + hipCheckErrors("reduce1"); if (dimgridRed > 1) { reduceSum << <1, dimblockRed, MAXTHREADS*sizeof(float) >> >(d_aux_small, d_norm2, dimgridRed); - cudaDeviceSynchronize(); - cudaCheckErrors("reduce2"); - cudaMemcpy(&sumnorm2, d_norm2, sizeof(float), cudaMemcpyDeviceToHost); - cudaCheckErrors("cudaMemcpy"); + hipDeviceSynchronize(); + hipCheckErrors("reduce2"); + hipMemcpy(&sumnorm2, d_norm2, sizeof(float), hipMemcpyDeviceToHost); + hipCheckErrors("hipMemcpy"); } else { - cudaMemcpy(&sumnorm2, d_aux_small, sizeof(float), cudaMemcpyDeviceToHost); - cudaCheckErrors("cudaMemcpy"); + hipMemcpy(&sumnorm2, d_aux_small, sizeof(float), hipMemcpyDeviceToHost); + hipCheckErrors("hipMemcpy"); } // mexPrintf("alpha/sqrt(sumnorm2): %f\n",alpha/sqrt(sumnorm2)); //MULTIPLY HYPERPARAMETER sqrt(sumnorm2) multiplyArrayScalar<<<60,MAXTHREADS>>>(d_dimgTV,alpha/sqrt(sumnorm2), total_pixels); - cudaDeviceSynchronize(); - cudaCheckErrors("Multiplication error"); + hipDeviceSynchronize(); + hipCheckErrors("Multiplication error"); //SUBSTRACT GRADIENT substractArrays <<<60,MAXTHREADS>>>(d_image,d_dimgTV, total_pixels); - cudaDeviceSynchronize(); - cudaCheckErrors("Substraction error"); + hipDeviceSynchronize(); + hipCheckErrors("Substraction error"); // mexPrintf("Final update is nan: %s\n",isnan_cuda(d_image,total_pixels) ? "true" : "false"); // mexPrintf("\n"); sumnorm2=0; } - cudaCheckErrors("TV minimization"); + hipCheckErrors("TV minimization"); - cudaMemcpy(dst, d_image, mem_size, cudaMemcpyDeviceToHost); - cudaCheckErrors("Copy result back"); + hipMemcpy(dst, d_image, mem_size, hipMemcpyDeviceToHost); + hipCheckErrors("Copy result back"); - cudaFree(d_image); - cudaFree(d_dpiccsTV); - cudaFree(d_aux_image); - cudaFree(d_aux_small); - cudaFree(d_prior); - cudaFree(d_norm2); + hipFree(d_image); + hipFree(d_dpiccsTV); + hipFree(d_aux_image); + hipFree(d_aux_small); + hipFree(d_prior); + hipFree(d_norm2); - cudaCheckErrors("Memory free"); - // Do NOT cudaDeviceReset() here - it destroys the whole host + hipCheckErrors("Memory free"); + // Do NOT hipDeviceReset() here - it destroys the whole host // process's primary CUDA context (CuPy/PyTorch state included), // not just this function's resources. See the matching note in // GD_TV.cu; all buffers are already freed explicitly above. - // cudaDeviceReset(); + // hipDeviceReset(); } diff --git a/Common/CUDA/RandomNumberGenerator.cu b/Common/CUDA/RandomNumberGenerator.cu index d7d1224a..11caf932 100644 --- a/Common/CUDA/RandomNumberGenerator.cu +++ b/Common/CUDA/RandomNumberGenerator.cu @@ -1,193 +1,193 @@ -/*------------------------------------------------------------------------- - * - * CUDA functions for random number generator - * - * Adds noise of Poisson and normal distribution to the input. - * - * CODE by Tomoyuki SADAKANE - * --------------------------------------------------------------------------- - * --------------------------------------------------------------------------- - * Copyright (c) 2015, University of Bath and CERN- European Organization for - * Nuclear Research - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * --------------------------------------------------------------------------- - * - * Contact: tigre.toolbox@gmail.com - * Codes : https://github.com/CERN/TIGRE - * --------------------------------------------------------------------------- - */ - -#include -#include -#include -#include -#include - -#include "gpuUtils.hpp" -#include "RandomNumberGenerator.hpp" - -#define cudaCheckErrors(msg) \ -do { \ - cudaError_t __err = cudaGetLastError(); \ - if (__err != cudaSuccess) { \ - mexPrintf("%s \n",msg);\ - cudaDeviceReset();\ - mexErrMsgIdAndTxt("RandomNumberGenerator:",cudaGetErrorString(__err));\ - } \ -} while (0) - - -__global__ void setup_kernel(curandState *state) { - int idx = threadIdx.x + blockIdx.x * blockDim.x; - /* Each thread gets same seed, a different sequence number, no offset */ - curand_init(1234, idx, 0, &state[idx]); -} - -__global__ void GeneratePoisson(curandState *state, const float* pfIn, size_t uiLen, float* pfOut) { - int idx = threadIdx.x + blockIdx.x * blockDim.x; - /* Copy state to local memory for efficiency */ - curandState localState = state[idx]; - int iIter = (uiLen + blockDim.x*gridDim.x - 1)/(blockDim.x*gridDim.x); - for (int iI = 0; iI < iIter; ++iI) { - size_t uiPos = (size_t)blockDim.x*gridDim.x*iI+idx; - if (uiPos < uiLen) { - /* Poisson */ - unsigned int uiPoisson = curand_poisson(&localState, pfIn[uiPos]); - pfOut[uiPos] = (float)uiPoisson; - } - } - /* Copy state back to global memory */ - state[idx] = localState; -} - -__global__ void GeneratePoissonAddGaussian(curandState *state, - const float* pfIn, - size_t uiLen, - float fGaussMu, - float fGaussSigma, - float* pfOut) -{ - int idx = threadIdx.x + blockIdx.x * blockDim.x; - /* Copy state to local memory for efficiency */ - curandState localState = state[idx]; - int iIter = (uiLen + blockDim.x*gridDim.x - 1)/(blockDim.x*gridDim.x); - for (int iI = 0; iI < iIter; ++iI) { - size_t uiPos = (size_t)blockDim.x*gridDim.x*iI+idx; - if (uiPos < uiLen) { - /* Poisson */ - unsigned int uiPoisson = curand_poisson(&localState, pfIn[uiPos]); - /* Gaussian */ - float fNormal = curand_normal(&localState) * fGaussSigma + fGaussMu; - pfOut[uiPos] = fNormal + (float)uiPoisson; - } - } - /* Copy state back to global memory */ - state[idx] = localState; -} - - -template -void GetMinMax(const T_value* pfIn, size_t uiLen, T_value& tvMin, T_value& tvMax) { - tvMin = pfIn[0]; - tvMax = pfIn[0]; - T_value tvVal; - for (int iI = 1; iI < uiLen; ++iI) { - tvVal = pfIn[iI]; - if (tvMax < tvVal) { tvMax = tvVal; continue;} - if (tvMin > tvVal) { tvMin = tvVal; continue;} - } -} -void poisson_1d(const float* pfIn, size_t uiLen, float* pfOut, const GpuIds& gpuids) { - // printf("poisson_1d(pfIn = %p, uiLen = %zd, pfOut = %p)\n", pfIn, uiLen, pfOut); - float* d_pfIn = nullptr; - float* d_pfOut = nullptr; - cudaMalloc((void **)&d_pfIn, uiLen * sizeof(float)); - cudaCheckErrors("poisson_1d fail cudaMalloc 1"); - cudaMalloc((void **)&d_pfOut, uiLen * sizeof(float)); - cudaCheckErrors("poisson_1d fail cudaMalloc 2"); - cudaMemcpy(d_pfIn, pfIn, uiLen*sizeof(float), cudaMemcpyHostToDevice); - cudaCheckErrors("poisson_1d fail cudaMemcpy 1"); - - // float fMin, fMax; - // GetMinMax(pfIn, uiLen, fMin, fMax); - // printf("fMin, fMax = %f, %f\n", fMin, fMax); - curandState *curandStates = nullptr; - const int kiBlockDim = 1024; // Threads per Block - const int kiGridDim = 64;//(uiLen+kiBlockDim-1)/kiBlockDim; - cudaMalloc((void **)&curandStates, kiGridDim * kiBlockDim * sizeof(curandState)); - cudaCheckErrors("poisson_1d fail cudaMalloc 3"); - setup_kernel<<>>(curandStates); - GeneratePoisson<<>>(curandStates, d_pfIn, uiLen, d_pfOut); - cudaMemcpy(pfOut, d_pfOut, uiLen*sizeof(float), cudaMemcpyDeviceToHost); - cudaCheckErrors("poisson_1d fail cudaMemcpy 2"); - // GetMinMax(pfOut, uiLen, fMin, fMax); - // printf("fMin, fMax = %f, %f\n", fMin, fMax); - - cudaFree(d_pfIn); d_pfIn = nullptr; - cudaFree(d_pfOut); d_pfOut = nullptr; - cudaFree(curandStates); curandStates = nullptr; -} - -void poisson_gaussian_1d(const float* pfIn, - size_t uiLen, - float fGaussMu, - float fGaussSigma, - float* pfOut, - GpuIds& gpuids) -{ - // printf("poisson_gaussian_1d(pfIn = %p, uiLen = %zd, fGaussMu = %+f, fGaussSigma = %f, pfOut = %p)\n", pfIn, uiLen, fGaussMu, fGaussSigma, pfOut); - float* d_pfIn = nullptr; - float* d_pfOut = nullptr; - cudaMalloc((void **)&d_pfIn, uiLen * sizeof(float)); - cudaCheckErrors("poisson_gaussian_1d fail cudaMalloc 1"); - cudaMalloc((void **)&d_pfOut, uiLen * sizeof(float)); - cudaCheckErrors("poisson_gaussian_1d fail cudaMalloc 2"); - cudaMemcpy(d_pfIn, pfIn, uiLen*sizeof(float), cudaMemcpyHostToDevice); - cudaCheckErrors("poisson_gaussian_1d fail cudaMemcpy 1"); - - // float fMin, fMax; - // GetMinMax(pfIn, uiLen, fMin, fMax); - // printf("fMin, fMax = %f, %f\n", fMin, fMax); - curandState *curandStates = nullptr; - const int kiBlockDim = 64; // Threads per Block - const int kiGridDim = 64;//(uiLen+kiBlockDim-1)/kiBlockDim; - cudaMalloc((void **)&curandStates, kiGridDim * kiBlockDim * sizeof(curandState)); - cudaCheckErrors("poisson_gaussian_1d fail cudaMalloc 3"); - setup_kernel<<>>(curandStates); - GeneratePoissonAddGaussian<<>>(curandStates, d_pfIn, uiLen, fGaussMu, fGaussSigma, d_pfOut); - cudaMemcpy(pfOut, d_pfOut, uiLen*sizeof(float), cudaMemcpyDeviceToHost); - cudaCheckErrors("poisson_gaussian_1d fail cudaMemcpy 2"); - // GetMinMax(pfOut, uiLen, fMin, fMax); - // printf("fMin, fMax = %f, %f\n", fMin, fMax); - - - cudaFree(d_pfIn); d_pfIn = nullptr; - cudaFree(d_pfOut); d_pfOut = nullptr; - cudaFree(curandStates); curandStates = nullptr; -} +/*------------------------------------------------------------------------- + * + * CUDA functions for random number generator + * + * Adds noise of Poisson and normal distribution to the input. + * + * CODE by Tomoyuki SADAKANE + * --------------------------------------------------------------------------- + * --------------------------------------------------------------------------- + * Copyright (c) 2015, University of Bath and CERN- European Organization for + * Nuclear Research + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * --------------------------------------------------------------------------- + * + * Contact: tigre.toolbox@gmail.com + * Codes : https://github.com/CERN/TIGRE + * --------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include + +#include "gpuUtils.hpp" +#include "RandomNumberGenerator.hpp" + +#define hipCheckErrors(msg) \ +do { \ + hipError_t __err = hipGetLastError(); \ + if (__err != hipSuccess) { \ + mexPrintf("%s \n",msg);\ + hipDeviceReset();\ + mexErrMsgIdAndTxt("RandomNumberGenerator:",hipGetErrorString(__err));\ + } \ +} while (0) + + +__global__ void setup_kernel(curandState *state) { + int idx = threadIdx.x + blockIdx.x * blockDim.x; + /* Each thread gets same seed, a different sequence number, no offset */ + curand_init(1234, idx, 0, &state[idx]); +} + +__global__ void GeneratePoisson(curandState *state, const float* pfIn, size_t uiLen, float* pfOut) { + int idx = threadIdx.x + blockIdx.x * blockDim.x; + /* Copy state to local memory for efficiency */ + curandState localState = state[idx]; + int iIter = (uiLen + blockDim.x*gridDim.x - 1)/(blockDim.x*gridDim.x); + for (int iI = 0; iI < iIter; ++iI) { + size_t uiPos = (size_t)blockDim.x*gridDim.x*iI+idx; + if (uiPos < uiLen) { + /* Poisson */ + unsigned int uiPoisson = curand_poisson(&localState, pfIn[uiPos]); + pfOut[uiPos] = (float)uiPoisson; + } + } + /* Copy state back to global memory */ + state[idx] = localState; +} + +__global__ void GeneratePoissonAddGaussian(curandState *state, + const float* pfIn, + size_t uiLen, + float fGaussMu, + float fGaussSigma, + float* pfOut) +{ + int idx = threadIdx.x + blockIdx.x * blockDim.x; + /* Copy state to local memory for efficiency */ + curandState localState = state[idx]; + int iIter = (uiLen + blockDim.x*gridDim.x - 1)/(blockDim.x*gridDim.x); + for (int iI = 0; iI < iIter; ++iI) { + size_t uiPos = (size_t)blockDim.x*gridDim.x*iI+idx; + if (uiPos < uiLen) { + /* Poisson */ + unsigned int uiPoisson = curand_poisson(&localState, pfIn[uiPos]); + /* Gaussian */ + float fNormal = curand_normal(&localState) * fGaussSigma + fGaussMu; + pfOut[uiPos] = fNormal + (float)uiPoisson; + } + } + /* Copy state back to global memory */ + state[idx] = localState; +} + + +template +void GetMinMax(const T_value* pfIn, size_t uiLen, T_value& tvMin, T_value& tvMax) { + tvMin = pfIn[0]; + tvMax = pfIn[0]; + T_value tvVal; + for (int iI = 1; iI < uiLen; ++iI) { + tvVal = pfIn[iI]; + if (tvMax < tvVal) { tvMax = tvVal; continue;} + if (tvMin > tvVal) { tvMin = tvVal; continue;} + } +} +void poisson_1d(const float* pfIn, size_t uiLen, float* pfOut, const GpuIds& gpuids) { + // printf("poisson_1d(pfIn = %p, uiLen = %zd, pfOut = %p)\n", pfIn, uiLen, pfOut); + float* d_pfIn = nullptr; + float* d_pfOut = nullptr; + hipMalloc((void **)&d_pfIn, uiLen * sizeof(float)); + hipCheckErrors("poisson_1d fail hipMalloc 1"); + hipMalloc((void **)&d_pfOut, uiLen * sizeof(float)); + hipCheckErrors("poisson_1d fail hipMalloc 2"); + hipMemcpy(d_pfIn, pfIn, uiLen*sizeof(float), hipMemcpyHostToDevice); + hipCheckErrors("poisson_1d fail hipMemcpy 1"); + + // float fMin, fMax; + // GetMinMax(pfIn, uiLen, fMin, fMax); + // printf("fMin, fMax = %f, %f\n", fMin, fMax); + curandState *curandStates = nullptr; + const int kiBlockDim = 1024; // Threads per Block + const int kiGridDim = 64;//(uiLen+kiBlockDim-1)/kiBlockDim; + hipMalloc((void **)&curandStates, kiGridDim * kiBlockDim * sizeof(curandState)); + hipCheckErrors("poisson_1d fail hipMalloc 3"); + setup_kernel<<>>(curandStates); + GeneratePoisson<<>>(curandStates, d_pfIn, uiLen, d_pfOut); + hipMemcpy(pfOut, d_pfOut, uiLen*sizeof(float), hipMemcpyDeviceToHost); + hipCheckErrors("poisson_1d fail hipMemcpy 2"); + // GetMinMax(pfOut, uiLen, fMin, fMax); + // printf("fMin, fMax = %f, %f\n", fMin, fMax); + + hipFree(d_pfIn); d_pfIn = nullptr; + hipFree(d_pfOut); d_pfOut = nullptr; + hipFree(curandStates); curandStates = nullptr; +} + +void poisson_gaussian_1d(const float* pfIn, + size_t uiLen, + float fGaussMu, + float fGaussSigma, + float* pfOut, + GpuIds& gpuids) +{ + // printf("poisson_gaussian_1d(pfIn = %p, uiLen = %zd, fGaussMu = %+f, fGaussSigma = %f, pfOut = %p)\n", pfIn, uiLen, fGaussMu, fGaussSigma, pfOut); + float* d_pfIn = nullptr; + float* d_pfOut = nullptr; + hipMalloc((void **)&d_pfIn, uiLen * sizeof(float)); + hipCheckErrors("poisson_gaussian_1d fail hipMalloc 1"); + hipMalloc((void **)&d_pfOut, uiLen * sizeof(float)); + hipCheckErrors("poisson_gaussian_1d fail hipMalloc 2"); + hipMemcpy(d_pfIn, pfIn, uiLen*sizeof(float), hipMemcpyHostToDevice); + hipCheckErrors("poisson_gaussian_1d fail hipMemcpy 1"); + + // float fMin, fMax; + // GetMinMax(pfIn, uiLen, fMin, fMax); + // printf("fMin, fMax = %f, %f\n", fMin, fMax); + curandState *curandStates = nullptr; + const int kiBlockDim = 64; // Threads per Block + const int kiGridDim = 64;//(uiLen+kiBlockDim-1)/kiBlockDim; + hipMalloc((void **)&curandStates, kiGridDim * kiBlockDim * sizeof(curandState)); + hipCheckErrors("poisson_gaussian_1d fail hipMalloc 3"); + setup_kernel<<>>(curandStates); + GeneratePoissonAddGaussian<<>>(curandStates, d_pfIn, uiLen, fGaussMu, fGaussSigma, d_pfOut); + hipMemcpy(pfOut, d_pfOut, uiLen*sizeof(float), hipMemcpyDeviceToHost); + hipCheckErrors("poisson_gaussian_1d fail hipMemcpy 2"); + // GetMinMax(pfOut, uiLen, fMin, fMax); + // printf("fMin, fMax = %f, %f\n", fMin, fMax); + + + hipFree(d_pfIn); d_pfIn = nullptr; + hipFree(d_pfOut); d_pfOut = nullptr; + hipFree(curandStates); curandStates = nullptr; +} diff --git a/Common/CUDA/RandomNumberGenerator.hpp b/Common/CUDA/RandomNumberGenerator.hpp index 4ba68d8d..f77ce31c 100644 --- a/Common/CUDA/RandomNumberGenerator.hpp +++ b/Common/CUDA/RandomNumberGenerator.hpp @@ -1,49 +1,49 @@ -/*------------------------------------------------------------------------- - * - * Header CUDA functions for random number generator - * - * Adds noise of Poisson and normal distribution to the input. - * - * CODE by Tomoyuki SADAKANE - * --------------------------------------------------------------------------- - * --------------------------------------------------------------------------- - * Copyright (c) 2015, University of Bath and CERN- European Organization for - * Nuclear Research - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * --------------------------------------------------------------------------- - * - * Contact: tigre.toolbox@gmail.com - * Codes : https://github.com/CERN/TIGRE - * --------------------------------------------------------------------------- - */ - -#include "TIGRE_common.hpp" -#include "GpuIds.hpp" -void poisson_1d(const float* pfIn, size_t uiLen, float* pfOut, const GpuIds& gpuids); -void poisson_gaussian_1d(const float* pfPoissonL, size_t uiLen, float fGaussMu, float fGaussSigma, float* pfOut, GpuIds& gpuids); +/*------------------------------------------------------------------------- + * + * Header CUDA functions for random number generator + * + * Adds noise of Poisson and normal distribution to the input. + * + * CODE by Tomoyuki SADAKANE + * --------------------------------------------------------------------------- + * --------------------------------------------------------------------------- + * Copyright (c) 2015, University of Bath and CERN- European Organization for + * Nuclear Research + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * --------------------------------------------------------------------------- + * + * Contact: tigre.toolbox@gmail.com + * Codes : https://github.com/CERN/TIGRE + * --------------------------------------------------------------------------- + */ + +#include "TIGRE_common.hpp" +#include "GpuIds.hpp" +void poisson_1d(const float* pfIn, size_t uiLen, float* pfOut, const GpuIds& gpuids); +void poisson_gaussian_1d(const float* pfPoissonL, size_t uiLen, float fGaussMu, float fGaussSigma, float* pfOut, GpuIds& gpuids); diff --git a/Common/CUDA/Siddon_projection.cu b/Common/CUDA/Siddon_projection.cu index 2a025f8c..faa000d1 100644 --- a/Common/CUDA/Siddon_projection.cu +++ b/Common/CUDA/Siddon_projection.cu @@ -1,859 +1,859 @@ -/*------------------------------------------------------------------------- - * - * CUDA functions for ray-voxel intersection based projection - * - * This file has the necessary fucntiosn to perform X-ray CBCT projection - * operation given a geaometry, angles and image. It usesthe so-called - * Jacobs algorithm to compute efficiently the length of the x-rays over - * voxel space. - * - * CODE by Ander Biguri - * Sepideh Hatamikia (arbitrary rotation) - * --------------------------------------------------------------------------- - * --------------------------------------------------------------------------- - * Copyright (c) 2015, University of Bath and CERN- European Organization for - * Nuclear Research - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * --------------------------------------------------------------------------- - * - * Contact: tigre.toolbox@gmail.com - * Codes : https://github.com/CERN/TIGRE - * --------------------------------------------------------------------------- - */ - -#include -#include -#include -#include "Siddon_projection.hpp" -#include "TIGRE_common.hpp" -#include - -#define cudaCheckErrors(msg) \ -do { \ - cudaError_t __err = cudaGetLastError(); \ - if (__err != cudaSuccess) { \ - mexPrintf("%s \n",msg);\ - mexErrMsgIdAndTxt("Ax:Siddon_projection",cudaGetErrorString(__err));\ - } \ -} while (0) - - -#define MAXTREADS 1024 -#define PROJ_PER_BLOCK 9 -#define PIXEL_SIZE_BLOCK 9 - /*GEOMETRY DEFINITION - * - * Detector plane, behind - * |-----------------------------| - * | | - * | | - * | | - * | | - * | +--------+ | - * | / /| | - * A Z | / / |*D | - * | | +--------+ | | - * | | | | | | - * | | | *O | + | - * --->y | | | / | - * / | | |/ | - * V X | +--------+ | - * |-----------------------------| - * - * *S - * - * - * - * - * - **/ - - void CreateTexture(const GpuIds& gpuids,const float* imagedata,Geometry geo,cudaArray** d_cuArrTex, cudaTextureObject_t *texImage,bool alloc); - -__constant__ Point3D projParamsArrayDev[4*PROJ_PER_BLOCK]; // Dev means it is on device - - -__global__ void vecAddInPlace(float *a, float *b, unsigned long n) -{ - int idx = blockIdx.x*blockDim.x+threadIdx.x; - // Make sure we do not go out of bounds - if (idx < n) - a[idx] = a[idx] + b[idx]; -} - -__global__ void kernelPixelDetector( Geometry geo, - float* detector, - const int currProjSetNumber, - const int totalNoOfProjections, - cudaTextureObject_t tex){ - - - unsigned long long u = blockIdx.x * blockDim.x + threadIdx.x; - unsigned long long v = blockIdx.y * blockDim.y + threadIdx.y; - unsigned long long projNumber=threadIdx.z; - - - if (u>= geo.nDetecU || v>= geo.nDetecV || projNumber>=PROJ_PER_BLOCK) - return; - -#if IS_FOR_MATLAB_TIGRE - size_t idx = (size_t)(u * (unsigned long long)geo.nDetecV + v)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; -#else - size_t idx = (size_t)(v * (unsigned long long)geo.nDetecU + u)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; -#endif - unsigned long indAlpha = currProjSetNumber*PROJ_PER_BLOCK+projNumber; // This is the ABSOLUTE projection number in the projection array (for a given GPU) - - if(indAlpha>=totalNoOfProjections) - return; - - Point3D uvOrigin = projParamsArrayDev[4*projNumber]; // 6*projNumber because we have 6 Point3D values per projection - Point3D deltaU = projParamsArrayDev[4*projNumber+1]; - Point3D deltaV = projParamsArrayDev[4*projNumber+2]; - Point3D source = projParamsArrayDev[4*projNumber+3]; - - /////// Get coordinates XYZ of pixel UV - unsigned long pixelV = geo.nDetecV-v-1; - unsigned long pixelU = u; - Point3D pixel1D; - pixel1D.x=(uvOrigin.x+pixelU*deltaU.x+pixelV*deltaV.x); - pixel1D.y=(uvOrigin.y+pixelU*deltaU.y+pixelV*deltaV.y); - pixel1D.z=(uvOrigin.z+pixelU*deltaU.z+pixelV*deltaV.z); - /////// - // Siddon's ray-voxel intersection, optimized as in doi=10.1.1.55.7516 - ////// - // Also called Jacobs algorithms - Point3D ray; - // vector of Xray - ray.x=pixel1D.x-source.x; - ray.y=pixel1D.y-source.y; - ray.z=pixel1D.z-source.z; - float eps=0.001; - ray.x=(fabsf(ray.x) Nvoxel+1 - - axm=fminf(__fdividef(-source.x,ray.x),__fdividef(geo.nVoxelX-source.x,ray.x)); - aym=fminf(__fdividef(-source.y,ray.y),__fdividef(geo.nVoxelY-source.y,ray.y)); - azm=fminf(__fdividef(-source.z,ray.z),__fdividef(geo.nVoxelZ-source.z,ray.z)); - axM=fmaxf(__fdividef(-source.x,ray.x),__fdividef(geo.nVoxelX-source.x,ray.x)); - ayM=fmaxf(__fdividef(-source.y,ray.y),__fdividef(geo.nVoxelY-source.y,ray.y)); - azM=fmaxf(__fdividef(-source.z,ray.z),__fdividef(geo.nVoxelZ-source.z,ray.z)); - - float am=fmaxf(fmaxf(axm,aym),azm); - float aM=fminf(fminf(axM,ayM),azM); - - // line intersects voxel space -> am=aM) - detector[idx]=0; - - // Compute max/min image INDEX for intersection eq(11-19) - // Discussion about ternary operator in CUDA: https://stackoverflow.com/questions/7104384/in-cuda-why-is-a-b010-more-efficient-than-an-if-else-version - float imin,imax,jmin,jmax,kmin,kmax; - // for X - if( source.x(tex, i, j, k); - i=i+iu; - ac=ax; - ax+=axu; - }else if(ay==aminc){ - sum+=(ay-ac)*tex3D(tex, i, j, k); - j=j+ju; - ac=ay; - ay+=ayu; - }else if(az==aminc){ - sum+=(az-ac)*tex3D(tex, i, j, k); - k=k+ku; - ac=az; - az+=azu; - } - aminc=fminf(fminf(ax,ay),az); - } - detector[idx]=sum*maxlength; -} - - -int siddon_ray_projection(float* img, Geometry geo, float** result,float const * const angles,int nangles, const GpuIds& gpuids){ - // Prepare for MultiGPU - int deviceCount = gpuids.GetLength(); - cudaCheckErrors("Device query fail"); - if (deviceCount == 0) { - mexErrMsgIdAndTxt("Ax:Siddon_projection:GPUselect","There are no available device(s) that support CUDA\n"); - } - // - // CODE assumes - // 1.-All available devices are usable by this code - // 2.-All available devices are equal, they are the same machine (warning thrown) - // Check the available devices, and if they are the same - if (!gpuids.AreEqualDevices()) { - mexWarnMsgIdAndTxt("Ax:Siddon_projection:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); - } - int dev; - - // Check free memory - size_t mem_GPU_global; - checkFreeMemory(gpuids, &mem_GPU_global); - - size_t mem_image= (unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY*(unsigned long long)geo.nVoxelZ*sizeof(float); - size_t mem_proj= (unsigned long long)geo.nDetecU*(unsigned long long)geo.nDetecV*sizeof(float); - - // Does everything fit in the GPUs? - const bool fits_in_memory = mem_image+2*PROJ_PER_BLOCK*mem_proj= 9020 - cudaDeviceGetAttribute(&isHostRegisterSupported,cudaDevAttrHostRegisterSupported,gpuids[0]); -#endif - // empirical testing shows that when the image split is smaller than 1 (also implies the image is not very big), the time to - // pin the memory is greater than the lost time in Synchronously launching the memcpys. This is only worth it when the image is too big. -#ifndef NO_PINNED_MEMORY - if (isHostRegisterSupported & (splits>1 |deviceCount>1)){ - cudaHostRegister(img, (size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geo.nVoxelZ*(size_t)sizeof(float),cudaHostRegisterPortable); - } -#endif - cudaCheckErrors("Error pinning memory"); - - - - // auxiliary variables - Point3D source, deltaU, deltaV, uvOrigin; - Point3D* projParamsArrayHost; - cudaMallocHost((void**)&projParamsArrayHost,4*PROJ_PER_BLOCK*sizeof(Point3D)); - cudaCheckErrors("Error allocating auxiliary constant memory"); - - // Create Streams for overlapping memcopy and compute - int nStreams=deviceCount*2; - cudaStream_t* stream=(cudaStream_t*)malloc(nStreams*sizeof(cudaStream_t));; - - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - for (int i = 0; i < 2; ++i){ - cudaStreamCreate(&stream[i+dev*2]); - - } - } - cudaCheckErrors("Stream creation fail"); - - int nangles_device=(nangles+deviceCount-1)/deviceCount; - int nangles_last_device=(nangles-(deviceCount-1)*nangles_device); - unsigned int noOfKernelCalls = (nangles_device+PROJ_PER_BLOCK-1)/PROJ_PER_BLOCK; // We'll take care of bounds checking inside the loop if nalpha is not divisible by PROJ_PER_BLOCK - unsigned int noOfKernelCallsLastDev = (nangles_last_device+PROJ_PER_BLOCK-1)/PROJ_PER_BLOCK; // we will use this in the memory management. - int projection_this_block; - cudaTextureObject_t *texImg = new cudaTextureObject_t[deviceCount]; - cudaArray **d_cuArrTex = new cudaArray*[deviceCount]; - - for (unsigned int sp=0;sp=nangles) - break; - if ((i*PROJ_PER_BLOCK+j)>=nangles_device) - break; - geoArray[sp].alpha=angles[proj_global*3]; - geoArray[sp].theta=angles[proj_global*3+1]; - geoArray[sp].psi =angles[proj_global*3+2]; - - - //precomute distances for faster execution - //Precompute per angle constant stuff for speed - computeDeltas_Siddon(geoArray[sp],proj_global, &uvOrigin, &deltaU, &deltaV, &source); - //Ray tracing! - projParamsArrayHost[4*j]=uvOrigin; // 6*j because we have 6 Point3D values per projection - projParamsArrayHost[4*j+1]=deltaU; - projParamsArrayHost[4*j+2]=deltaV; - projParamsArrayHost[4*j+3]=source; - - } - cudaMemcpyToSymbolAsync(projParamsArrayDev, projParamsArrayHost, sizeof(Point3D)*4*PROJ_PER_BLOCK,0,cudaMemcpyHostToDevice,stream[dev*2]); - cudaStreamSynchronize(stream[dev*2]); - cudaCheckErrors("kernel fail"); - kernelPixelDetector<<>>(geoArray[sp],dProjection[(i%2)+dev*2],i,nangles_device,texImg[dev]); - } - - - // Now that the computation is happening, we need to either prepare the memory for - // combining of the projections (splits>1) and start removing previous results. - - - // If our image does not fit in memory then we need to make sure we accumulate previous results too. - // This is done in 2 steps: - // 1)copy previous results back into GPU - // 2)accumulate with current results - // The code to take them out is the same as when there are no splits needed - if( !fits_in_memory&&sp>0) - { - // 1) grab previous results and put them in the auxiliary variable dProjection_accum - for (dev = 0; dev < deviceCount; dev++) - { - cudaSetDevice(gpuids[dev]); - //Global index of FIRST projection on this set on this GPU - proj_global=i*PROJ_PER_BLOCK+dev*nangles_device; - if(proj_global>=nangles) - break; - - // Unless its the last projection set, we have PROJ_PER_BLOCK angles. Otherwise... - if(i+1==noOfKernelCalls) //is it the last block? - projection_this_block=min(nangles_device-(noOfKernelCalls-1)*PROJ_PER_BLOCK, //the remaining angles that this GPU had to do (almost never PROJ_PER_BLOCK) - nangles-proj_global); //or whichever amount is left to finish all (this is for the last GPU) - else - projection_this_block=PROJ_PER_BLOCK; - - cudaMemcpyAsync(dProjection_accum[(i%2)+dev*2], result[proj_global], projection_this_block*geo.nDetecV*geo.nDetecU*sizeof(float), cudaMemcpyHostToDevice,stream[dev*2+1]); - } - // 2) take the results from current compute call and add it to the code in execution. - for (dev = 0; dev < deviceCount; dev++) - { - cudaSetDevice(gpuids[dev]); - //Global index of FIRST projection on this set on this GPU - proj_global=i*PROJ_PER_BLOCK+dev*nangles_device; - if(proj_global>=nangles) - break; - - // Unless its the last projection set, we have PROJ_PER_BLOCK angles. Otherwise... - if(i+1==noOfKernelCalls) //is it the last block? - projection_this_block=min(nangles_device-(noOfKernelCalls-1)*PROJ_PER_BLOCK, //the remaining angles that this GPU had to do (almost never PROJ_PER_BLOCK) - nangles-proj_global); //or whichever amount is left to finish all (this is for the last GPU) - else - projection_this_block=PROJ_PER_BLOCK; - - cudaStreamSynchronize(stream[dev*2+1]); // wait until copy is finished - vecAddInPlace<<<(geo.nDetecU*geo.nDetecV*projection_this_block+MAXTREADS-1)/MAXTREADS,MAXTREADS,0,stream[dev*2]>>>(dProjection[(i%2)+dev*2],dProjection_accum[(i%2)+dev*2],(unsigned long)geo.nDetecU*geo.nDetecV*projection_this_block); - } - } // end accumulation case, where the image needs to be split - - // Now, lets get out the projections from the previous execution of the kernels. - if (i>0){ - for (dev = 0; dev < deviceCount; dev++) - { - cudaSetDevice(gpuids[dev]); - //Global index of FIRST projection on previous set on this GPU - proj_global=(i-1)*PROJ_PER_BLOCK+dev*nangles_device; - if (dev+1==deviceCount) { //is it the last device? - // projections assigned to this device is >=nangles_device-(deviceCount-1) and < nangles_device - if (i-1 < noOfKernelCallsLastDev) { - // The previous set(block) was not empty. - projection_this_block=min(PROJ_PER_BLOCK, nangles-proj_global); - } - else { - // The previous set was empty. - // This happens if deviceCount > PROJ_PER_BLOCK+1. - // e.g. PROJ_PER_BLOCK = 9, deviceCount = 11, nangles = 199. - // e.g. PROJ_PER_BLOCK = 1, deviceCount = 3, nangles = 7. - break; - } - } - else { - projection_this_block=PROJ_PER_BLOCK; - } - cudaMemcpyAsync(result[proj_global], dProjection[(int)(!(i%2))+dev*2], projection_this_block*geo.nDetecV*geo.nDetecU*sizeof(float), cudaMemcpyDeviceToHost,stream[dev*2+1]); - } - } - // Make sure Computation on kernels has finished before we launch the next batch. - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaStreamSynchronize(stream[dev*2]); - } - } - - - // We still have the last set of projections to get out of GPUs - for (dev = 0; dev < deviceCount; dev++) - { - cudaSetDevice(gpuids[dev]); - //Global index of FIRST projection on this set on this GPU - proj_global=(noOfKernelCalls-1)*PROJ_PER_BLOCK+dev*nangles_device; - if(proj_global>=nangles) - break; - // How many projections are left here? - projection_this_block=min(nangles_device-(noOfKernelCalls-1)*PROJ_PER_BLOCK, //the remaining angles that this GPU had to do (almost never PROJ_PER_BLOCK) - nangles-proj_global); //or whichever amount is left to finish all (this is for the last GPU) - - cudaDeviceSynchronize(); //Not really necessary, but just in case, we los nothing. - cudaCheckErrors("Error at copying the last set of projections out (or in the previous copy)"); - cudaMemcpyAsync(result[proj_global], dProjection[(int)(!(noOfKernelCalls%2))+dev*2], projection_this_block*geo.nDetecV*geo.nDetecU*sizeof(float), cudaMemcpyDeviceToHost,stream[dev*2+1]); - } - // Make sure everyone has done their bussiness before the next image split: - cudaDeviceSynchronize(); - } // End image split loop. - - cudaCheckErrors("Main loop fail"); - /////////////////////////////////////////////////////////////////////// - /////////////////////////////////////////////////////////////////////// - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDestroyTextureObject(texImg[dev]); - cudaFreeArray(d_cuArrTex[dev]); - } - delete[] texImg; texImg = 0; - delete[] d_cuArrTex; d_cuArrTex = 0; - // Freeing Stage - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaFree(dProjection[dev*2]); - cudaFree(dProjection[dev*2+1]); - - } - free(dProjection); - - if(!fits_in_memory){ - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaFree(dProjection_accum[dev*2]); - cudaFree(dProjection_accum[dev*2+1]); - - } - free(dProjection_accum); - } - freeGeoArray(splits,geoArray); - cudaFreeHost(projParamsArrayHost); - - - for (int i = 0; i < nStreams; ++i) - cudaStreamDestroy(stream[i]) ; -#ifndef NO_PINNED_MEMORY - if (isHostRegisterSupported & (splits>1 |deviceCount>1)){ - cudaHostUnregister(img); - } - cudaCheckErrors("cudaFree fail"); -#endif - //cudaDeviceReset(); - return 0; -} - - - - -void CreateTexture(const GpuIds& gpuids,const float* imagedata,Geometry geo,cudaArray** d_cuArrTex, cudaTextureObject_t *texImage,bool alloc) -{ - //size_t size_image=geo.nVoxelX*geo.nVoxelY*geo.nVoxelZ; - const cudaExtent extent = make_cudaExtent(geo.nVoxelX, geo.nVoxelY, geo.nVoxelZ); - const unsigned int num_devices = gpuids.GetLength(); - if(alloc){ - for (unsigned int dev = 0; dev < num_devices; dev++){ - cudaSetDevice(gpuids[dev]); - - //cudaArray Descriptor - cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(); - //cuda Array - cudaMalloc3DArray(&d_cuArrTex[dev], &channelDesc, extent); - } - } - for (unsigned int dev = 0; dev < num_devices; dev++){ - cudaSetDevice(gpuids[dev]); - cudaMemcpy3DParms copyParams = {0}; - //Array creation - copyParams.srcPtr = make_cudaPitchedPtr((void *)imagedata, extent.width*sizeof(float), extent.width, extent.height); - copyParams.dstArray = d_cuArrTex[dev]; - copyParams.extent = extent; - copyParams.kind = cudaMemcpyHostToDevice; - cudaMemcpy3DAsync(©Params); - } - for (unsigned int dev = 0; dev < num_devices; dev++){ - cudaSetDevice(gpuids[dev]); - cudaResourceDesc texRes; - memset(&texRes, 0, sizeof(cudaResourceDesc)); - texRes.resType = cudaResourceTypeArray; - texRes.res.array.array = d_cuArrTex[dev]; - cudaTextureDesc texDescr; - memset(&texDescr, 0, sizeof(cudaTextureDesc)); - texDescr.normalizedCoords = false; - texDescr.filterMode = cudaFilterModePoint; - texDescr.addressMode[0] = cudaAddressModeBorder; - texDescr.addressMode[1] = cudaAddressModeBorder; - texDescr.addressMode[2] = cudaAddressModeBorder; - texDescr.readMode = cudaReadModeElementType; - cudaCreateTextureObject(&texImage[dev], &texRes, &texDescr, NULL); - - } - for (unsigned int dev = 0; dev < num_devices; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - } - cudaCheckErrors("Texture object creation fail"); -} - -/* This code generates the geometries needed to split the image properly in - * cases where the entire image does not fit in the memory of the GPU - **/ -void splitImage(unsigned int splits,Geometry geo,Geometry* geoArray, unsigned int nangles){ - - unsigned long splitsize=(geo.nVoxelZ+splits-1)/splits;// ceil if not divisible - for(unsigned int sp=0;spx; - auxPoint.y=point->y; - auxPoint.z=point->z; - - point->x=cos(geo.dRoll[i])*cos(geo.dPitch[i])*auxPoint.x - +(cos(geo.dRoll[i])*sin(geo.dPitch[i])*sin(geo.dYaw[i]) - sin(geo.dRoll[i])*cos(geo.dYaw[i]))*auxPoint.y - +(cos(geo.dRoll[i])*sin(geo.dPitch[i])*cos(geo.dYaw[i]) + sin(geo.dRoll[i])*sin(geo.dYaw[i]))*auxPoint.z; - - point->y=sin(geo.dRoll[i])*cos(geo.dPitch[i])*auxPoint.x - +(sin(geo.dRoll[i])*sin(geo.dPitch[i])*sin(geo.dYaw[i]) + cos(geo.dRoll[i])*cos(geo.dYaw[i]))*auxPoint.y - +(sin(geo.dRoll[i])*sin(geo.dPitch[i])*cos(geo.dYaw[i]) - cos(geo.dRoll[i])*sin(geo.dYaw[i]))*auxPoint.z; - - point->z=-sin(geo.dPitch[i])*auxPoint.x - +cos(geo.dPitch[i])*sin(geo.dYaw[i])*auxPoint.y - +cos(geo.dPitch[i])*cos(geo.dYaw[i])*auxPoint.z; - -} -void eulerZYZ(Geometry geo, Point3D* point){ - Point3D auxPoint; - auxPoint.x=point->x; - auxPoint.y=point->y; - auxPoint.z=point->z; - - point->x=(+cos(geo.alpha)*cos(geo.theta)*cos(geo.psi)-sin(geo.alpha)*sin(geo.psi))*auxPoint.x+ - (-cos(geo.alpha)*cos(geo.theta)*sin(geo.psi)-sin(geo.alpha)*cos(geo.psi))*auxPoint.y+ - cos(geo.alpha)*sin(geo.theta)*auxPoint.z; - - point->y=(+sin(geo.alpha)*cos(geo.theta)*cos(geo.psi)+cos(geo.alpha)*sin(geo.psi))*auxPoint.x+ - (-sin(geo.alpha)*cos(geo.theta)*sin(geo.psi)+cos(geo.alpha)*cos(geo.psi))*auxPoint.y+ - sin(geo.alpha)*sin(geo.theta)*auxPoint.z; - - point->z=-sin(geo.theta)*cos(geo.psi)*auxPoint.x+ - sin(geo.theta)*sin(geo.psi)*auxPoint.y+ - cos(geo.theta)*auxPoint.z; - - -} -//______________________________________________________________________________ -// -// Function: freeGeoArray -// -// Description: Frees the memory from the geometry array for multiGPU. -//______________________________________________________________________________ -void freeGeoArray(unsigned int splits,Geometry* geoArray){ - for(unsigned int sp=0;sp +#include +#include +#include "Siddon_projection.hpp" +#include "TIGRE_common.hpp" +#include + +#define hipCheckErrors(msg) \ +do { \ + hipError_t __err = hipGetLastError(); \ + if (__err != hipSuccess) { \ + mexPrintf("%s \n",msg);\ + mexErrMsgIdAndTxt("Ax:Siddon_projection",hipGetErrorString(__err));\ + } \ +} while (0) + + +#define MAXTREADS 1024 +#define PROJ_PER_BLOCK 9 +#define PIXEL_SIZE_BLOCK 9 + /*GEOMETRY DEFINITION + * + * Detector plane, behind + * |-----------------------------| + * | | + * | | + * | | + * | | + * | +--------+ | + * | / /| | + * A Z | / / |*D | + * | | +--------+ | | + * | | | | | | + * | | | *O | + | + * --->y | | | / | + * / | | |/ | + * V X | +--------+ | + * |-----------------------------| + * + * *S + * + * + * + * + * + **/ + + void CreateTexture(const GpuIds& gpuids,const float* imagedata,Geometry geo,hipArray** d_cuArrTex, hipTextureObject_t *texImage,bool alloc); + +__constant__ Point3D projParamsArrayDev[4*PROJ_PER_BLOCK]; // Dev means it is on device + + +__global__ void vecAddInPlace(float *a, float *b, unsigned long n) +{ + int idx = blockIdx.x*blockDim.x+threadIdx.x; + // Make sure we do not go out of bounds + if (idx < n) + a[idx] = a[idx] + b[idx]; +} + +__global__ void kernelPixelDetector( Geometry geo, + float* detector, + const int currProjSetNumber, + const int totalNoOfProjections, + hipTextureObject_t tex){ + + + unsigned long long u = blockIdx.x * blockDim.x + threadIdx.x; + unsigned long long v = blockIdx.y * blockDim.y + threadIdx.y; + unsigned long long projNumber=threadIdx.z; + + + if (u>= geo.nDetecU || v>= geo.nDetecV || projNumber>=PROJ_PER_BLOCK) + return; + +#if IS_FOR_MATLAB_TIGRE + size_t idx = (size_t)(u * (unsigned long long)geo.nDetecV + v)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; +#else + size_t idx = (size_t)(v * (unsigned long long)geo.nDetecU + u)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; +#endif + unsigned long indAlpha = currProjSetNumber*PROJ_PER_BLOCK+projNumber; // This is the ABSOLUTE projection number in the projection array (for a given GPU) + + if(indAlpha>=totalNoOfProjections) + return; + + Point3D uvOrigin = projParamsArrayDev[4*projNumber]; // 6*projNumber because we have 6 Point3D values per projection + Point3D deltaU = projParamsArrayDev[4*projNumber+1]; + Point3D deltaV = projParamsArrayDev[4*projNumber+2]; + Point3D source = projParamsArrayDev[4*projNumber+3]; + + /////// Get coordinates XYZ of pixel UV + unsigned long pixelV = geo.nDetecV-v-1; + unsigned long pixelU = u; + Point3D pixel1D; + pixel1D.x=(uvOrigin.x+pixelU*deltaU.x+pixelV*deltaV.x); + pixel1D.y=(uvOrigin.y+pixelU*deltaU.y+pixelV*deltaV.y); + pixel1D.z=(uvOrigin.z+pixelU*deltaU.z+pixelV*deltaV.z); + /////// + // Siddon's ray-voxel intersection, optimized as in doi=10.1.1.55.7516 + ////// + // Also called Jacobs algorithms + Point3D ray; + // vector of Xray + ray.x=pixel1D.x-source.x; + ray.y=pixel1D.y-source.y; + ray.z=pixel1D.z-source.z; + float eps=0.001; + ray.x=(fabsf(ray.x) Nvoxel+1 + + axm=fminf(__fdividef(-source.x,ray.x),__fdividef(geo.nVoxelX-source.x,ray.x)); + aym=fminf(__fdividef(-source.y,ray.y),__fdividef(geo.nVoxelY-source.y,ray.y)); + azm=fminf(__fdividef(-source.z,ray.z),__fdividef(geo.nVoxelZ-source.z,ray.z)); + axM=fmaxf(__fdividef(-source.x,ray.x),__fdividef(geo.nVoxelX-source.x,ray.x)); + ayM=fmaxf(__fdividef(-source.y,ray.y),__fdividef(geo.nVoxelY-source.y,ray.y)); + azM=fmaxf(__fdividef(-source.z,ray.z),__fdividef(geo.nVoxelZ-source.z,ray.z)); + + float am=fmaxf(fmaxf(axm,aym),azm); + float aM=fminf(fminf(axM,ayM),azM); + + // line intersects voxel space -> am=aM) + detector[idx]=0; + + // Compute max/min image INDEX for intersection eq(11-19) + // Discussion about ternary operator in CUDA: https://stackoverflow.com/questions/7104384/in-cuda-why-is-a-b010-more-efficient-than-an-if-else-version + float imin,imax,jmin,jmax,kmin,kmax; + // for X + if( source.x(tex, i, j, k); + i=i+iu; + ac=ax; + ax+=axu; + }else if(ay==aminc){ + sum+=(ay-ac)*tex3D(tex, i, j, k); + j=j+ju; + ac=ay; + ay+=ayu; + }else if(az==aminc){ + sum+=(az-ac)*tex3D(tex, i, j, k); + k=k+ku; + ac=az; + az+=azu; + } + aminc=fminf(fminf(ax,ay),az); + } + detector[idx]=sum*maxlength; +} + + +int siddon_ray_projection(float* img, Geometry geo, float** result,float const * const angles,int nangles, const GpuIds& gpuids){ + // Prepare for MultiGPU + int deviceCount = gpuids.GetLength(); + hipCheckErrors("Device query fail"); + if (deviceCount == 0) { + mexErrMsgIdAndTxt("Ax:Siddon_projection:GPUselect","There are no available device(s) that support CUDA\n"); + } + // + // CODE assumes + // 1.-All available devices are usable by this code + // 2.-All available devices are equal, they are the same machine (warning thrown) + // Check the available devices, and if they are the same + if (!gpuids.AreEqualDevices()) { + mexWarnMsgIdAndTxt("Ax:Siddon_projection:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); + } + int dev; + + // Check free memory + size_t mem_GPU_global; + checkFreeMemory(gpuids, &mem_GPU_global); + + size_t mem_image= (unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY*(unsigned long long)geo.nVoxelZ*sizeof(float); + size_t mem_proj= (unsigned long long)geo.nDetecU*(unsigned long long)geo.nDetecV*sizeof(float); + + // Does everything fit in the GPUs? + const bool fits_in_memory = mem_image+2*PROJ_PER_BLOCK*mem_proj= 9020 + hipDeviceGetAttribute(&isHostRegisterSupported,hipDevAttrHostRegisterSupported,gpuids[0]); +#endif + // empirical testing shows that when the image split is smaller than 1 (also implies the image is not very big), the time to + // pin the memory is greater than the lost time in Synchronously launching the memcpys. This is only worth it when the image is too big. +#ifndef NO_PINNED_MEMORY + if (isHostRegisterSupported & (splits>1 |deviceCount>1)){ + hipHostRegister(img, (size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geo.nVoxelZ*(size_t)sizeof(float),hipHostRegisterPortable); + } +#endif + hipCheckErrors("Error pinning memory"); + + + + // auxiliary variables + Point3D source, deltaU, deltaV, uvOrigin; + Point3D* projParamsArrayHost; + hipMallocHost((void**)&projParamsArrayHost,4*PROJ_PER_BLOCK*sizeof(Point3D)); + hipCheckErrors("Error allocating auxiliary constant memory"); + + // Create Streams for overlapping memcopy and compute + int nStreams=deviceCount*2; + hipStream_t* stream=(hipStream_t*)malloc(nStreams*sizeof(hipStream_t));; + + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + for (int i = 0; i < 2; ++i){ + hipStreamCreate(&stream[i+dev*2]); + + } + } + hipCheckErrors("Stream creation fail"); + + int nangles_device=(nangles+deviceCount-1)/deviceCount; + int nangles_last_device=(nangles-(deviceCount-1)*nangles_device); + unsigned int noOfKernelCalls = (nangles_device+PROJ_PER_BLOCK-1)/PROJ_PER_BLOCK; // We'll take care of bounds checking inside the loop if nalpha is not divisible by PROJ_PER_BLOCK + unsigned int noOfKernelCallsLastDev = (nangles_last_device+PROJ_PER_BLOCK-1)/PROJ_PER_BLOCK; // we will use this in the memory management. + int projection_this_block; + hipTextureObject_t *texImg = new hipTextureObject_t[deviceCount]; + hipArray **d_cuArrTex = new hipArray*[deviceCount]; + + for (unsigned int sp=0;sp=nangles) + break; + if ((i*PROJ_PER_BLOCK+j)>=nangles_device) + break; + geoArray[sp].alpha=angles[proj_global*3]; + geoArray[sp].theta=angles[proj_global*3+1]; + geoArray[sp].psi =angles[proj_global*3+2]; + + + //precomute distances for faster execution + //Precompute per angle constant stuff for speed + computeDeltas_Siddon(geoArray[sp],proj_global, &uvOrigin, &deltaU, &deltaV, &source); + //Ray tracing! + projParamsArrayHost[4*j]=uvOrigin; // 6*j because we have 6 Point3D values per projection + projParamsArrayHost[4*j+1]=deltaU; + projParamsArrayHost[4*j+2]=deltaV; + projParamsArrayHost[4*j+3]=source; + + } + hipMemcpyToSymbolAsync(projParamsArrayDev, projParamsArrayHost, sizeof(Point3D)*4*PROJ_PER_BLOCK,0,hipMemcpyHostToDevice,stream[dev*2]); + hipStreamSynchronize(stream[dev*2]); + hipCheckErrors("kernel fail"); + kernelPixelDetector<<>>(geoArray[sp],dProjection[(i%2)+dev*2],i,nangles_device,texImg[dev]); + } + + + // Now that the computation is happening, we need to either prepare the memory for + // combining of the projections (splits>1) and start removing previous results. + + + // If our image does not fit in memory then we need to make sure we accumulate previous results too. + // This is done in 2 steps: + // 1)copy previous results back into GPU + // 2)accumulate with current results + // The code to take them out is the same as when there are no splits needed + if( !fits_in_memory&&sp>0) + { + // 1) grab previous results and put them in the auxiliary variable dProjection_accum + for (dev = 0; dev < deviceCount; dev++) + { + hipSetDevice(gpuids[dev]); + //Global index of FIRST projection on this set on this GPU + proj_global=i*PROJ_PER_BLOCK+dev*nangles_device; + if(proj_global>=nangles) + break; + + // Unless its the last projection set, we have PROJ_PER_BLOCK angles. Otherwise... + if(i+1==noOfKernelCalls) //is it the last block? + projection_this_block=min(nangles_device-(noOfKernelCalls-1)*PROJ_PER_BLOCK, //the remaining angles that this GPU had to do (almost never PROJ_PER_BLOCK) + nangles-proj_global); //or whichever amount is left to finish all (this is for the last GPU) + else + projection_this_block=PROJ_PER_BLOCK; + + hipMemcpyAsync(dProjection_accum[(i%2)+dev*2], result[proj_global], projection_this_block*geo.nDetecV*geo.nDetecU*sizeof(float), hipMemcpyHostToDevice,stream[dev*2+1]); + } + // 2) take the results from current compute call and add it to the code in execution. + for (dev = 0; dev < deviceCount; dev++) + { + hipSetDevice(gpuids[dev]); + //Global index of FIRST projection on this set on this GPU + proj_global=i*PROJ_PER_BLOCK+dev*nangles_device; + if(proj_global>=nangles) + break; + + // Unless its the last projection set, we have PROJ_PER_BLOCK angles. Otherwise... + if(i+1==noOfKernelCalls) //is it the last block? + projection_this_block=min(nangles_device-(noOfKernelCalls-1)*PROJ_PER_BLOCK, //the remaining angles that this GPU had to do (almost never PROJ_PER_BLOCK) + nangles-proj_global); //or whichever amount is left to finish all (this is for the last GPU) + else + projection_this_block=PROJ_PER_BLOCK; + + hipStreamSynchronize(stream[dev*2+1]); // wait until copy is finished + vecAddInPlace<<<(geo.nDetecU*geo.nDetecV*projection_this_block+MAXTREADS-1)/MAXTREADS,MAXTREADS,0,stream[dev*2]>>>(dProjection[(i%2)+dev*2],dProjection_accum[(i%2)+dev*2],(unsigned long)geo.nDetecU*geo.nDetecV*projection_this_block); + } + } // end accumulation case, where the image needs to be split + + // Now, lets get out the projections from the previous execution of the kernels. + if (i>0){ + for (dev = 0; dev < deviceCount; dev++) + { + hipSetDevice(gpuids[dev]); + //Global index of FIRST projection on previous set on this GPU + proj_global=(i-1)*PROJ_PER_BLOCK+dev*nangles_device; + if (dev+1==deviceCount) { //is it the last device? + // projections assigned to this device is >=nangles_device-(deviceCount-1) and < nangles_device + if (i-1 < noOfKernelCallsLastDev) { + // The previous set(block) was not empty. + projection_this_block=min(PROJ_PER_BLOCK, nangles-proj_global); + } + else { + // The previous set was empty. + // This happens if deviceCount > PROJ_PER_BLOCK+1. + // e.g. PROJ_PER_BLOCK = 9, deviceCount = 11, nangles = 199. + // e.g. PROJ_PER_BLOCK = 1, deviceCount = 3, nangles = 7. + break; + } + } + else { + projection_this_block=PROJ_PER_BLOCK; + } + hipMemcpyAsync(result[proj_global], dProjection[(int)(!(i%2))+dev*2], projection_this_block*geo.nDetecV*geo.nDetecU*sizeof(float), hipMemcpyDeviceToHost,stream[dev*2+1]); + } + } + // Make sure Computation on kernels has finished before we launch the next batch. + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipStreamSynchronize(stream[dev*2]); + } + } + + + // We still have the last set of projections to get out of GPUs + for (dev = 0; dev < deviceCount; dev++) + { + hipSetDevice(gpuids[dev]); + //Global index of FIRST projection on this set on this GPU + proj_global=(noOfKernelCalls-1)*PROJ_PER_BLOCK+dev*nangles_device; + if(proj_global>=nangles) + break; + // How many projections are left here? + projection_this_block=min(nangles_device-(noOfKernelCalls-1)*PROJ_PER_BLOCK, //the remaining angles that this GPU had to do (almost never PROJ_PER_BLOCK) + nangles-proj_global); //or whichever amount is left to finish all (this is for the last GPU) + + hipDeviceSynchronize(); //Not really necessary, but just in case, we los nothing. + hipCheckErrors("Error at copying the last set of projections out (or in the previous copy)"); + hipMemcpyAsync(result[proj_global], dProjection[(int)(!(noOfKernelCalls%2))+dev*2], projection_this_block*geo.nDetecV*geo.nDetecU*sizeof(float), hipMemcpyDeviceToHost,stream[dev*2+1]); + } + // Make sure everyone has done their bussiness before the next image split: + hipDeviceSynchronize(); + } // End image split loop. + + hipCheckErrors("Main loop fail"); + /////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////// + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDestroyTextureObject(texImg[dev]); + hipFreeArray(d_cuArrTex[dev]); + } + delete[] texImg; texImg = 0; + delete[] d_cuArrTex; d_cuArrTex = 0; + // Freeing Stage + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipFree(dProjection[dev*2]); + hipFree(dProjection[dev*2+1]); + + } + free(dProjection); + + if(!fits_in_memory){ + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipFree(dProjection_accum[dev*2]); + hipFree(dProjection_accum[dev*2+1]); + + } + free(dProjection_accum); + } + freeGeoArray(splits,geoArray); + hipFreeHost(projParamsArrayHost); + + + for (int i = 0; i < nStreams; ++i) + hipStreamDestroy(stream[i]) ; +#ifndef NO_PINNED_MEMORY + if (isHostRegisterSupported & (splits>1 |deviceCount>1)){ + hipHostUnregister(img); + } + hipCheckErrors("hipFree fail"); +#endif + //hipDeviceReset(); + return 0; +} + + + + +void CreateTexture(const GpuIds& gpuids,const float* imagedata,Geometry geo,hipArray** d_cuArrTex, hipTextureObject_t *texImage,bool alloc) +{ + //size_t size_image=geo.nVoxelX*geo.nVoxelY*geo.nVoxelZ; + const hipExtent extent = make_cudaExtent(geo.nVoxelX, geo.nVoxelY, geo.nVoxelZ); + const unsigned int num_devices = gpuids.GetLength(); + if(alloc){ + for (unsigned int dev = 0; dev < num_devices; dev++){ + hipSetDevice(gpuids[dev]); + + //hipArray Descriptor + hipChannelFormatDesc channelDesc = hipCreateChannelDesc(); + //cuda Array + hipMalloc3DArray(&d_cuArrTex[dev], &channelDesc, extent); + } + } + for (unsigned int dev = 0; dev < num_devices; dev++){ + hipSetDevice(gpuids[dev]); + hipMemcpy3DParms copyParams = {0}; + //Array creation + copyParams.srcPtr = make_cudaPitchedPtr((void *)imagedata, extent.width*sizeof(float), extent.width, extent.height); + copyParams.dstArray = d_cuArrTex[dev]; + copyParams.extent = extent; + copyParams.kind = hipMemcpyHostToDevice; + hipMemcpy3DAsync(©Params); + } + for (unsigned int dev = 0; dev < num_devices; dev++){ + hipSetDevice(gpuids[dev]); + hipResourceDesc texRes; + memset(&texRes, 0, sizeof(hipResourceDesc)); + texRes.resType = hipResourceTypeArray; + texRes.res.array.array = d_cuArrTex[dev]; + hipTextureDesc texDescr; + memset(&texDescr, 0, sizeof(hipTextureDesc)); + texDescr.normalizedCoords = false; + texDescr.filterMode = hipFilterModePoint; + texDescr.addressMode[0] = hipAddressModeBorder; + texDescr.addressMode[1] = hipAddressModeBorder; + texDescr.addressMode[2] = hipAddressModeBorder; + texDescr.readMode = hipReadModeElementType; + hipCreateTextureObject(&texImage[dev], &texRes, &texDescr, NULL); + + } + for (unsigned int dev = 0; dev < num_devices; dev++){ + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + } + hipCheckErrors("Texture object creation fail"); +} + +/* This code generates the geometries needed to split the image properly in + * cases where the entire image does not fit in the memory of the GPU + **/ +void splitImage(unsigned int splits,Geometry geo,Geometry* geoArray, unsigned int nangles){ + + unsigned long splitsize=(geo.nVoxelZ+splits-1)/splits;// ceil if not divisible + for(unsigned int sp=0;spx; + auxPoint.y=point->y; + auxPoint.z=point->z; + + point->x=cos(geo.dRoll[i])*cos(geo.dPitch[i])*auxPoint.x + +(cos(geo.dRoll[i])*sin(geo.dPitch[i])*sin(geo.dYaw[i]) - sin(geo.dRoll[i])*cos(geo.dYaw[i]))*auxPoint.y + +(cos(geo.dRoll[i])*sin(geo.dPitch[i])*cos(geo.dYaw[i]) + sin(geo.dRoll[i])*sin(geo.dYaw[i]))*auxPoint.z; + + point->y=sin(geo.dRoll[i])*cos(geo.dPitch[i])*auxPoint.x + +(sin(geo.dRoll[i])*sin(geo.dPitch[i])*sin(geo.dYaw[i]) + cos(geo.dRoll[i])*cos(geo.dYaw[i]))*auxPoint.y + +(sin(geo.dRoll[i])*sin(geo.dPitch[i])*cos(geo.dYaw[i]) - cos(geo.dRoll[i])*sin(geo.dYaw[i]))*auxPoint.z; + + point->z=-sin(geo.dPitch[i])*auxPoint.x + +cos(geo.dPitch[i])*sin(geo.dYaw[i])*auxPoint.y + +cos(geo.dPitch[i])*cos(geo.dYaw[i])*auxPoint.z; + +} +void eulerZYZ(Geometry geo, Point3D* point){ + Point3D auxPoint; + auxPoint.x=point->x; + auxPoint.y=point->y; + auxPoint.z=point->z; + + point->x=(+cos(geo.alpha)*cos(geo.theta)*cos(geo.psi)-sin(geo.alpha)*sin(geo.psi))*auxPoint.x+ + (-cos(geo.alpha)*cos(geo.theta)*sin(geo.psi)-sin(geo.alpha)*cos(geo.psi))*auxPoint.y+ + cos(geo.alpha)*sin(geo.theta)*auxPoint.z; + + point->y=(+sin(geo.alpha)*cos(geo.theta)*cos(geo.psi)+cos(geo.alpha)*sin(geo.psi))*auxPoint.x+ + (-sin(geo.alpha)*cos(geo.theta)*sin(geo.psi)+cos(geo.alpha)*cos(geo.psi))*auxPoint.y+ + sin(geo.alpha)*sin(geo.theta)*auxPoint.z; + + point->z=-sin(geo.theta)*cos(geo.psi)*auxPoint.x+ + sin(geo.theta)*sin(geo.psi)*auxPoint.y+ + cos(geo.theta)*auxPoint.z; + + +} +//______________________________________________________________________________ +// +// Function: freeGeoArray +// +// Description: Frees the memory from the geometry array for multiGPU. +//______________________________________________________________________________ +void freeGeoArray(unsigned int splits,Geometry* geoArray){ + for(unsigned int sp=0;sp -#include -#include -#include "Siddon_projection_parallel.hpp" -#include "TIGRE_common.hpp" -#include - -#define cudaCheckErrors(msg) \ -do { \ - cudaError_t __err = cudaGetLastError(); \ - if (__err != cudaSuccess) { \ - mexPrintf("%s \n",msg);\ - mexErrMsgIdAndTxt("TIGRE:CUDA:Ax",cudaGetErrorString(__err));\ - } \ -} while (0) - - -// Declare the texture reference. -void CreateTextureParallel(float* image,Geometry geo,cudaArray** d_cuArrTex, cudaTextureObject_t *texImage,cudaStream_t* stream); - - -#define MAXTREADS 1024 -#define PROJ_PER_BLOCK 9 -#define PIXEL_SIZE_BLOCK 9 -/*GEOMETRY DEFINITION - * - * Detector plane, behind - * |-----------------------------| - * | | - * | | - * | | - * | | - * | +--------+ | - * | / /| | - * A Z | / / |*D | - * | | +--------+ | | - * | | | | | | - * | | | *O | + | - * --->y | | | / | - * / | | |/ | - * V X | +--------+ | - * |-----------------------------| - * - * *S - * - * - * - * - * - **/ - - -__constant__ Point3D projParamsArrayDev[4*PROJ_PER_BLOCK]; // Dev means it is on device - - -__global__ void kernelPixelDetector_parallel( Geometry geo, - float* detector, const int currProjSetNumber, const int totalNoOfProjections, cudaTextureObject_t tex){ - - unsigned long long u = blockIdx.x * blockDim.x + threadIdx.x; - unsigned long long v = blockIdx.y * blockDim.y + threadIdx.y; - unsigned long long projNumber=threadIdx.z; - - if (u>= geo.nDetecU || v>= geo.nDetecV || projNumber>=PROJ_PER_BLOCK) - return; - - unsigned long indAlpha = currProjSetNumber*PROJ_PER_BLOCK+projNumber; // This is the ABSOLUTE projection number in the projection array - - -#if IS_FOR_MATLAB_TIGRE - size_t idx = (size_t)(u * (unsigned long long)geo.nDetecV + v)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; -#else - size_t idx = (size_t)(v * (unsigned long long)geo.nDetecU + u)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; -#endif - - if(indAlpha>=totalNoOfProjections) - return; - - Point3D uvOrigin = projParamsArrayDev[4*projNumber]; // 6*projNumber because we have 6 Point3D values per projection - Point3D deltaU = projParamsArrayDev[4*projNumber+1]; - Point3D deltaV = projParamsArrayDev[4*projNumber+2]; - Point3D source = projParamsArrayDev[4*projNumber+3]; - - - /////// Get coordinates XYZ of pixel UV - unsigned long pixelV = geo.nDetecV-v-1; - unsigned long pixelU = u; - Point3D pixel1D; - pixel1D.x=(uvOrigin.x+pixelU*deltaU.x+pixelV*deltaV.x); - pixel1D.y=(uvOrigin.y+pixelU*deltaU.y+pixelV*deltaV.y); - pixel1D.z=(uvOrigin.z+pixelU*deltaU.z+pixelV*deltaV.z); - - - source.x=(source.x+pixelU*deltaU.x+pixelV*deltaV.x); - source.y=(source.y+pixelU*deltaU.y+pixelV*deltaV.y); - source.z=(source.z+pixelU*deltaU.z+pixelV*deltaV.z); - /////// - // Siddon's ray-voxel intersection, optimized as in doi=10.1.1.55.7516 - ////// - Point3D ray; - // vector of Xray - ray.x=pixel1D.x-source.x; - ray.y=pixel1D.y-source.y; - ray.z=pixel1D.z-source.z; - // This variables are ommited because - // bx,by,bz ={0,0,0} - // dx,dy,dz ={1,1,1} - // compute parameter values for x-ray parametric equation. eq(3-10) - float axm,aym,azm; - float axM,ayM,azM; - - /************************************** - * - * - * Problem. In paralel beam, often ray.y or ray.x=0; - * This leads to infinities progpagating and breaking everything. - * - * We need to fix it. - * - ***************************************/ - - // In the paper Nx= number of X planes-> Nvoxel+1 - axm=fminf(-source.x/ray.x,(geo.nVoxelX-source.x)/ray.x); - aym=fminf(-source.y/ray.y,(geo.nVoxelY-source.y)/ray.y); -// azm=min(-source.z/ray.z,(geo.nVoxelZ-source.z)/ray.z); - axM=fmaxf(-source.x/ray.x,(geo.nVoxelX-source.x)/ray.x); - ayM=fmaxf(-source.y/ray.y,(geo.nVoxelY-source.y)/ray.y); -// azM=max(-source.z/ray.z,(geo.nVoxelZ-source.z)/ray.z); - float am=(fmaxf(axm,aym)); - float aM=(fminf(axM,ayM)); - - // line intersects voxel space -> am=aM) - detector[idx]=0.0f; - - // Compute max/min image INDEX for intersection eq(11-19) - // Discussion about ternary operator in CUDA: https://stackoverflow.com/questions/7104384/in-cuda-why-is-a-b010-more-efficient-than-an-if-else-version - float imin,imax,jmin,jmax; - // for X - if( source.x(tex, i, j, k);//(ax-ac)* - i=i+iu; - ac=ax; - ax+=axu; - }else if(ay==aminc){ - sum+=(ay-ac)*tex3D(tex, i, j, k);//(ay-ac)* - j=j+ju; - ac=ay; - ay+=ayu; -// }else if(az==aminc){ -// sum+=(az-ac)*tex3D(tex, i+0.5, j+0.5, k+0.5); -// k=k+ku; -// ac=az; -// az+=azu; - } - aminc=fminf(ay,ax); - } - detector[idx]=maxlength*sum; -} - - -int siddon_ray_projection_parallel(float* img, Geometry geo, float** result,float const * const angles,int nangles, const GpuIds& gpuids){ - - - - - - size_t num_bytes = (size_t)geo.nDetecU*(size_t)geo.nDetecV*(size_t)PROJ_PER_BLOCK* (size_t)sizeof(float); - float** dProjection=(float **)malloc(2*sizeof(float *)); - for (int i = 0; i < 2; ++i){ - cudaMalloc((void**)&dProjection[i], num_bytes); - cudaCheckErrors("cudaMalloc projections fail"); - } - int nStreams=2; - cudaStream_t* stream=(cudaStream_t*)malloc(nStreams*sizeof(cudaStream_t)); - - for (int i = 0; i < 2; ++i){ - cudaStreamCreate(&stream[i]); - } - - - - // Texture object variables - cudaTextureObject_t *texImg = 0; - cudaArray **d_cuArrTex = 0; - texImg =(cudaTextureObject_t*)malloc(1*sizeof(cudaTextureObject_t)); - d_cuArrTex =(cudaArray**)malloc(1*sizeof(cudaArray*)); - - CreateTextureParallel(img,geo,&d_cuArrTex[0], &texImg [0],stream); - cudaCheckErrors("Texture allocation fail"); - //Done! Image put into texture memory. - - - - Point3D source, deltaU, deltaV, uvOrigin; - - - Point3D* projParamsArrayHost; - cudaMallocHost((void**)&projParamsArrayHost,4*PROJ_PER_BLOCK*sizeof(Point3D)); - - // 16x16 gave the best performance empirically - // Funnily that makes it compatible with most GPUs..... - int divU,divV,divangle; - divU=PIXEL_SIZE_BLOCK; - divV=PIXEL_SIZE_BLOCK; - - dim3 numBlocks((geo.nDetecU+divU-1)/divU,(geo.nDetecV+divV-1)/divV,1); - - dim3 threadsPerBlock(divU,divV,PROJ_PER_BLOCK); - - unsigned int proj_global; - unsigned int noOfKernelCalls = (nangles+PROJ_PER_BLOCK-1)/PROJ_PER_BLOCK; // We'll take care of bounds checking inside the loop if nalpha is not divisible by PROJ_PER_BLOCK - unsigned int i; - for ( i=0; i=nangles) - break; - geo.alpha=angles[proj_global*3]; - geo.theta=angles[proj_global*3+1]; - geo.psi =angles[proj_global*3+2]; - - //precomute distances for faster execution - //Precompute per angle constant stuff for speed - computeDeltas_Siddon_parallel(geo,geo.alpha,proj_global, &uvOrigin, &deltaU, &deltaV, &source); - //Ray tracing! - projParamsArrayHost[4*j]=uvOrigin; // 6*j because we have 6 Point3D values per projection - projParamsArrayHost[4*j+1]=deltaU; - projParamsArrayHost[4*j+2]=deltaV; - projParamsArrayHost[4*j+3]=source; - - } - - cudaMemcpyToSymbolAsync(projParamsArrayDev, projParamsArrayHost, sizeof(Point3D)*4*PROJ_PER_BLOCK,0,cudaMemcpyHostToDevice,stream[0]); - cudaStreamSynchronize(stream[0]); - kernelPixelDetector_parallel<<>>(geo,dProjection[(int)i%2==0],i,nangles,texImg[0]); - // copy result to host - if (i>0) - cudaMemcpyAsync(result[i*PROJ_PER_BLOCK-PROJ_PER_BLOCK],dProjection[(int)i%2!=0], num_bytes, cudaMemcpyDeviceToHost,stream[1]); - } - cudaDeviceSynchronize(); - - int lastangles=nangles-(i-1)*PROJ_PER_BLOCK; - cudaMemcpyAsync(result[(i-1)*PROJ_PER_BLOCK],dProjection[(int)(i-1)%2==0], lastangles*geo.nDetecV*geo.nDetecU*sizeof(float), cudaMemcpyDeviceToHost,stream[1]); - - - - cudaDestroyTextureObject(texImg[0]); - cudaFreeArray(d_cuArrTex[0]); - free(texImg); texImg = 0; - free(d_cuArrTex); d_cuArrTex = 0; - cudaCheckErrors("Unbind fail"); - cudaFree(dProjection[0]); - cudaFree(dProjection[1]); - free(dProjection); - cudaFreeHost(projParamsArrayHost); - cudaCheckErrors("cudaFree d_imagedata fail"); - - - for (int i = 0; i < 2; ++i){ - cudaStreamDestroy(stream[i]); - } -// cudaDeviceReset(); - return 0; -} - - - -/* This code precomputes The location of the source and the Delta U and delta V (in the warped space) - * to compute the locations of the x-rays. While it seems verbose and overly-optimized, - * it does saves about 30% of each of the kernel calls. Thats something! - **/ -void computeDeltas_Siddon_parallel(Geometry geo, float angles,int i, Point3D* uvorigin, Point3D* deltaU, Point3D* deltaV, Point3D* source){ - Point3D S; - - S.x =geo.DSO[i]; S.y = geo.dDetecU*(0-((float)geo.nDetecU/2)+0.5); S.z = geo.dDetecV*(((float)geo.nDetecV/2)-0.5-0); - - //End point - Point3D P,Pu0,Pv0; - - P.x =-(geo.DSD[i]-geo.DSO[i]); P.y = geo.dDetecU*(0-((float)geo.nDetecU/2)+0.5); P.z = geo.dDetecV*(((float)geo.nDetecV/2)-0.5-0); - Pu0.x=-(geo.DSD[i]-geo.DSO[i]); Pu0.y= geo.dDetecU*(1-((float)geo.nDetecU/2)+0.5); Pu0.z= geo.dDetecV*(((float)geo.nDetecV/2)-0.5-0); - Pv0.x=-(geo.DSD[i]-geo.DSO[i]); Pv0.y= geo.dDetecU*(0-((float)geo.nDetecU/2)+0.5); Pv0.z= geo.dDetecV*(((float)geo.nDetecV/2)-0.5-1); - // Geometric trasnformations: - P.x=0;Pu0.x=0;Pv0.x=0; - - // Roll pitch yaw - rollPitchYaw(geo,i,&P); - rollPitchYaw(geo,i,&Pu0); - rollPitchYaw(geo,i,&Pv0); - //Now lets translate the points where they should be: - P.x=P.x-(geo.DSD[i]-geo.DSO[i]); - Pu0.x=Pu0.x-(geo.DSD[i]-geo.DSO[i]); - Pv0.x=Pv0.x-(geo.DSD[i]-geo.DSO[i]); - - S.x=0; - // Roll pitch yaw - rollPitchYaw(geo,i,&S); - //Now lets translate the points where they should be: - S.x=S.x+geo.DSO[i]; - - //1: Offset detector - - //P.x - P.y =P.y +geo.offDetecU[i]; P.z =P.z +geo.offDetecV[i]; - Pu0.y=Pu0.y+geo.offDetecU[i]; Pu0.z=Pu0.z+geo.offDetecV[i]; - Pv0.y=Pv0.y+geo.offDetecU[i]; Pv0.z=Pv0.z+geo.offDetecV[i]; - //S doesnt need to chagne - - - //3: Rotate (around z)! - Point3D Pfinal, Pfinalu0, Pfinalv0; - - Pfinal.x =P.x*cos(geo.alpha)-P.y*sin(geo.alpha); Pfinal.y =P.y*cos(geo.alpha)+P.x*sin(geo.alpha); Pfinal.z =P.z; - Pfinalu0.x=Pu0.x*cos(geo.alpha)-Pu0.y*sin(geo.alpha); Pfinalu0.y=Pu0.y*cos(geo.alpha)+Pu0.x*sin(geo.alpha); Pfinalu0.z=Pu0.z; - Pfinalv0.x=Pv0.x*cos(geo.alpha)-Pv0.y*sin(geo.alpha); Pfinalv0.y=Pv0.y*cos(geo.alpha)+Pv0.x*sin(geo.alpha); Pfinalv0.z=Pv0.z; - - Point3D S2; - S2.x=S.x*cos(geo.alpha)-S.y*sin(geo.alpha); - S2.y=S.y*cos(geo.alpha)+S.x*sin(geo.alpha); - S2.z=S.z; - - //2: Offset image (instead of offseting image, -offset everything else) - - Pfinal.x =Pfinal.x-geo.offOrigX[i]; Pfinal.y =Pfinal.y-geo.offOrigY[i]; Pfinal.z =Pfinal.z-geo.offOrigZ[i]; - Pfinalu0.x=Pfinalu0.x-geo.offOrigX[i]; Pfinalu0.y=Pfinalu0.y-geo.offOrigY[i]; Pfinalu0.z=Pfinalu0.z-geo.offOrigZ[i]; - Pfinalv0.x=Pfinalv0.x-geo.offOrigX[i]; Pfinalv0.y=Pfinalv0.y-geo.offOrigY[i]; Pfinalv0.z=Pfinalv0.z-geo.offOrigZ[i]; - S2.x=S2.x-geo.offOrigX[i]; S2.y=S2.y-geo.offOrigY[i]; S2.z=S2.z-geo.offOrigZ[i]; - - // As we want the (0,0,0) to be in a corner of the image, we need to translate everything (after rotation); - Pfinal.x =Pfinal.x+geo.sVoxelX/2; Pfinal.y =Pfinal.y+geo.sVoxelY/2; Pfinal.z =Pfinal.z +geo.sVoxelZ/2; - Pfinalu0.x=Pfinalu0.x+geo.sVoxelX/2; Pfinalu0.y=Pfinalu0.y+geo.sVoxelY/2; Pfinalu0.z=Pfinalu0.z+geo.sVoxelZ/2; - Pfinalv0.x=Pfinalv0.x+geo.sVoxelX/2; Pfinalv0.y=Pfinalv0.y+geo.sVoxelY/2; Pfinalv0.z=Pfinalv0.z+geo.sVoxelZ/2; - S2.x =S2.x+geo.sVoxelX/2; S2.y =S2.y+geo.sVoxelY/2; S2.z =S2.z +geo.sVoxelZ/2; - - //4. Scale everything so dVoxel==1 - Pfinal.x =Pfinal.x/geo.dVoxelX; Pfinal.y =Pfinal.y/geo.dVoxelY; Pfinal.z =Pfinal.z/geo.dVoxelZ; - Pfinalu0.x=Pfinalu0.x/geo.dVoxelX; Pfinalu0.y=Pfinalu0.y/geo.dVoxelY; Pfinalu0.z=Pfinalu0.z/geo.dVoxelZ; - Pfinalv0.x=Pfinalv0.x/geo.dVoxelX; Pfinalv0.y=Pfinalv0.y/geo.dVoxelY; Pfinalv0.z=Pfinalv0.z/geo.dVoxelZ; - S2.x =S2.x/geo.dVoxelX; S2.y =S2.y/geo.dVoxelY; S2.z =S2.z/geo.dVoxelZ; - - - - //5. apply COR. Wherever everything was, now its offesetd by a bit - float CORx, CORy; - CORx=-geo.COR[i]*sin(geo.alpha)/geo.dVoxelX; - CORy= geo.COR[i]*cos(geo.alpha)/geo.dVoxelY; - Pfinal.x+=CORx; Pfinal.y+=CORy; - Pfinalu0.x+=CORx; Pfinalu0.y+=CORy; - Pfinalv0.x+=CORx; Pfinalv0.y+=CORy; - S2.x+=CORx; S2.y+=CORy; - - // return - - *uvorigin=Pfinal; - - deltaU->x=Pfinalu0.x-Pfinal.x; - deltaU->y=Pfinalu0.y-Pfinal.y; - deltaU->z=Pfinalu0.z-Pfinal.z; - - deltaV->x=Pfinalv0.x-Pfinal.x; - deltaV->y=Pfinalv0.y-Pfinal.y; - deltaV->z=Pfinalv0.z-Pfinal.z; - - *source=S2; -} -void CreateTextureParallel(float* image,Geometry geo,cudaArray** d_cuArrTex, cudaTextureObject_t *texImage,cudaStream_t* stream){ //size_t size_image=geo.nVoxelX*geo.nVoxelY*geo.nVoxelZ; - - - const cudaExtent extent = make_cudaExtent(geo.nVoxelX, geo.nVoxelY, geo.nVoxelZ); - - //cudaArray Descriptor - cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(); - //cuda Array - cudaMalloc3DArray(&d_cuArrTex[0], &channelDesc, extent); - - - cudaMemcpy3DParms copyParams = {0}; - //Array creation - copyParams.srcPtr = make_cudaPitchedPtr((void *)image, extent.width*sizeof(float), extent.width, extent.height); - copyParams.dstArray = d_cuArrTex[0]; - copyParams.extent = extent; - copyParams.kind = cudaMemcpyHostToDevice; - cudaMemcpy3DAsync(©Params,stream[1]); - - - //Array creation End - - cudaResourceDesc texRes; - memset(&texRes, 0, sizeof(cudaResourceDesc)); - texRes.resType = cudaResourceTypeArray; - texRes.res.array.array = d_cuArrTex[0]; - cudaTextureDesc texDescr; - memset(&texDescr, 0, sizeof(cudaTextureDesc)); - texDescr.normalizedCoords = false; - texDescr.filterMode = cudaFilterModePoint; - texDescr.addressMode[0] = cudaAddressModeBorder; - texDescr.addressMode[1] = cudaAddressModeBorder; - texDescr.addressMode[2] = cudaAddressModeBorder; - texDescr.readMode = cudaReadModeElementType; - cudaCreateTextureObject(&texImage[0], &texRes, &texDescr, NULL); - -} - -#ifndef PROJECTION_HPP - -float maxDistanceCubeXY(Geometry geo, float alpha,int i){ - /////////// - // Compute initial "t" so we access safely as less as out of bounds as possible. - ////////// - - - float maxCubX,maxCubY; - // Forgetting Z, compute max distance: diagonal+offset - maxCubX=(geo.sVoxelX/2+ abs(geo.offOrigX[i]))/geo.dVoxelX; - maxCubY=(geo.sVoxelY/2+ abs(geo.offOrigY[i]))/geo.dVoxelY; - - return geo.DSO[i]/geo.dVoxelX-sqrt(maxCubX*maxCubX+maxCubY*maxCubY); - -} - -#endif +/*------------------------------------------------------------------------- + * + * CUDA functions for ray-voxel intersection based projection + * + * This file has the necessary fucntiosn to perform X-ray parallel projection + * operation given a geaometry, angles and image. It usesthe so-called + * Jacobs algorithm to compute efficiently the length of the x-rays over + * voxel space. Its called Siddon because Jacobs algorithm its just a small + * improvement over the traditional Siddons method. + * + * CODE by Ander Biguri + * + * --------------------------------------------------------------------------- + * --------------------------------------------------------------------------- + * Copyright (c) 2015, University of Bath and CERN- European Organization for + * Nuclear Research + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * --------------------------------------------------------------------------- + * + * Contact: tigre.toolbox@gmail.com + * Codes : https://github.com/CERN/TIGRE + * --------------------------------------------------------------------------- + */ + + +#include +#include +#include +#include "Siddon_projection_parallel.hpp" +#include "TIGRE_common.hpp" +#include + +#define hipCheckErrors(msg) \ +do { \ + hipError_t __err = hipGetLastError(); \ + if (__err != hipSuccess) { \ + mexPrintf("%s \n",msg);\ + mexErrMsgIdAndTxt("TIGRE:CUDA:Ax",hipGetErrorString(__err));\ + } \ +} while (0) + + +// Declare the texture reference. +void CreateTextureParallel(float* image,Geometry geo,hipArray** d_cuArrTex, hipTextureObject_t *texImage,hipStream_t* stream); + + +#define MAXTREADS 1024 +#define PROJ_PER_BLOCK 9 +#define PIXEL_SIZE_BLOCK 9 +/*GEOMETRY DEFINITION + * + * Detector plane, behind + * |-----------------------------| + * | | + * | | + * | | + * | | + * | +--------+ | + * | / /| | + * A Z | / / |*D | + * | | +--------+ | | + * | | | | | | + * | | | *O | + | + * --->y | | | / | + * / | | |/ | + * V X | +--------+ | + * |-----------------------------| + * + * *S + * + * + * + * + * + **/ + + +__constant__ Point3D projParamsArrayDev[4*PROJ_PER_BLOCK]; // Dev means it is on device + + +__global__ void kernelPixelDetector_parallel( Geometry geo, + float* detector, const int currProjSetNumber, const int totalNoOfProjections, hipTextureObject_t tex){ + + unsigned long long u = blockIdx.x * blockDim.x + threadIdx.x; + unsigned long long v = blockIdx.y * blockDim.y + threadIdx.y; + unsigned long long projNumber=threadIdx.z; + + if (u>= geo.nDetecU || v>= geo.nDetecV || projNumber>=PROJ_PER_BLOCK) + return; + + unsigned long indAlpha = currProjSetNumber*PROJ_PER_BLOCK+projNumber; // This is the ABSOLUTE projection number in the projection array + + +#if IS_FOR_MATLAB_TIGRE + size_t idx = (size_t)(u * (unsigned long long)geo.nDetecV + v)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; +#else + size_t idx = (size_t)(v * (unsigned long long)geo.nDetecU + u)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; +#endif + + if(indAlpha>=totalNoOfProjections) + return; + + Point3D uvOrigin = projParamsArrayDev[4*projNumber]; // 6*projNumber because we have 6 Point3D values per projection + Point3D deltaU = projParamsArrayDev[4*projNumber+1]; + Point3D deltaV = projParamsArrayDev[4*projNumber+2]; + Point3D source = projParamsArrayDev[4*projNumber+3]; + + + /////// Get coordinates XYZ of pixel UV + unsigned long pixelV = geo.nDetecV-v-1; + unsigned long pixelU = u; + Point3D pixel1D; + pixel1D.x=(uvOrigin.x+pixelU*deltaU.x+pixelV*deltaV.x); + pixel1D.y=(uvOrigin.y+pixelU*deltaU.y+pixelV*deltaV.y); + pixel1D.z=(uvOrigin.z+pixelU*deltaU.z+pixelV*deltaV.z); + + + source.x=(source.x+pixelU*deltaU.x+pixelV*deltaV.x); + source.y=(source.y+pixelU*deltaU.y+pixelV*deltaV.y); + source.z=(source.z+pixelU*deltaU.z+pixelV*deltaV.z); + /////// + // Siddon's ray-voxel intersection, optimized as in doi=10.1.1.55.7516 + ////// + Point3D ray; + // vector of Xray + ray.x=pixel1D.x-source.x; + ray.y=pixel1D.y-source.y; + ray.z=pixel1D.z-source.z; + // This variables are ommited because + // bx,by,bz ={0,0,0} + // dx,dy,dz ={1,1,1} + // compute parameter values for x-ray parametric equation. eq(3-10) + float axm,aym,azm; + float axM,ayM,azM; + + /************************************** + * + * + * Problem. In paralel beam, often ray.y or ray.x=0; + * This leads to infinities progpagating and breaking everything. + * + * We need to fix it. + * + ***************************************/ + + // In the paper Nx= number of X planes-> Nvoxel+1 + axm=fminf(-source.x/ray.x,(geo.nVoxelX-source.x)/ray.x); + aym=fminf(-source.y/ray.y,(geo.nVoxelY-source.y)/ray.y); +// azm=min(-source.z/ray.z,(geo.nVoxelZ-source.z)/ray.z); + axM=fmaxf(-source.x/ray.x,(geo.nVoxelX-source.x)/ray.x); + ayM=fmaxf(-source.y/ray.y,(geo.nVoxelY-source.y)/ray.y); +// azM=max(-source.z/ray.z,(geo.nVoxelZ-source.z)/ray.z); + float am=(fmaxf(axm,aym)); + float aM=(fminf(axM,ayM)); + + // line intersects voxel space -> am=aM) + detector[idx]=0.0f; + + // Compute max/min image INDEX for intersection eq(11-19) + // Discussion about ternary operator in CUDA: https://stackoverflow.com/questions/7104384/in-cuda-why-is-a-b010-more-efficient-than-an-if-else-version + float imin,imax,jmin,jmax; + // for X + if( source.x(tex, i, j, k);//(ax-ac)* + i=i+iu; + ac=ax; + ax+=axu; + }else if(ay==aminc){ + sum+=(ay-ac)*tex3D(tex, i, j, k);//(ay-ac)* + j=j+ju; + ac=ay; + ay+=ayu; +// }else if(az==aminc){ +// sum+=(az-ac)*tex3D(tex, i+0.5, j+0.5, k+0.5); +// k=k+ku; +// ac=az; +// az+=azu; + } + aminc=fminf(ay,ax); + } + detector[idx]=maxlength*sum; +} + + +int siddon_ray_projection_parallel(float* img, Geometry geo, float** result,float const * const angles,int nangles, const GpuIds& gpuids){ + + + + + + size_t num_bytes = (size_t)geo.nDetecU*(size_t)geo.nDetecV*(size_t)PROJ_PER_BLOCK* (size_t)sizeof(float); + float** dProjection=(float **)malloc(2*sizeof(float *)); + for (int i = 0; i < 2; ++i){ + hipMalloc((void**)&dProjection[i], num_bytes); + hipCheckErrors("hipMalloc projections fail"); + } + int nStreams=2; + hipStream_t* stream=(hipStream_t*)malloc(nStreams*sizeof(hipStream_t)); + + for (int i = 0; i < 2; ++i){ + hipStreamCreate(&stream[i]); + } + + + + // Texture object variables + hipTextureObject_t *texImg = 0; + hipArray **d_cuArrTex = 0; + texImg =(hipTextureObject_t*)malloc(1*sizeof(hipTextureObject_t)); + d_cuArrTex =(hipArray**)malloc(1*sizeof(hipArray*)); + + CreateTextureParallel(img,geo,&d_cuArrTex[0], &texImg [0],stream); + hipCheckErrors("Texture allocation fail"); + //Done! Image put into texture memory. + + + + Point3D source, deltaU, deltaV, uvOrigin; + + + Point3D* projParamsArrayHost; + hipMallocHost((void**)&projParamsArrayHost,4*PROJ_PER_BLOCK*sizeof(Point3D)); + + // 16x16 gave the best performance empirically + // Funnily that makes it compatible with most GPUs..... + int divU,divV,divangle; + divU=PIXEL_SIZE_BLOCK; + divV=PIXEL_SIZE_BLOCK; + + dim3 numBlocks((geo.nDetecU+divU-1)/divU,(geo.nDetecV+divV-1)/divV,1); + + dim3 threadsPerBlock(divU,divV,PROJ_PER_BLOCK); + + unsigned int proj_global; + unsigned int noOfKernelCalls = (nangles+PROJ_PER_BLOCK-1)/PROJ_PER_BLOCK; // We'll take care of bounds checking inside the loop if nalpha is not divisible by PROJ_PER_BLOCK + unsigned int i; + for ( i=0; i=nangles) + break; + geo.alpha=angles[proj_global*3]; + geo.theta=angles[proj_global*3+1]; + geo.psi =angles[proj_global*3+2]; + + //precomute distances for faster execution + //Precompute per angle constant stuff for speed + computeDeltas_Siddon_parallel(geo,geo.alpha,proj_global, &uvOrigin, &deltaU, &deltaV, &source); + //Ray tracing! + projParamsArrayHost[4*j]=uvOrigin; // 6*j because we have 6 Point3D values per projection + projParamsArrayHost[4*j+1]=deltaU; + projParamsArrayHost[4*j+2]=deltaV; + projParamsArrayHost[4*j+3]=source; + + } + + hipMemcpyToSymbolAsync(projParamsArrayDev, projParamsArrayHost, sizeof(Point3D)*4*PROJ_PER_BLOCK,0,hipMemcpyHostToDevice,stream[0]); + hipStreamSynchronize(stream[0]); + kernelPixelDetector_parallel<<>>(geo,dProjection[(int)i%2==0],i,nangles,texImg[0]); + // copy result to host + if (i>0) + hipMemcpyAsync(result[i*PROJ_PER_BLOCK-PROJ_PER_BLOCK],dProjection[(int)i%2!=0], num_bytes, hipMemcpyDeviceToHost,stream[1]); + } + hipDeviceSynchronize(); + + int lastangles=nangles-(i-1)*PROJ_PER_BLOCK; + hipMemcpyAsync(result[(i-1)*PROJ_PER_BLOCK],dProjection[(int)(i-1)%2==0], lastangles*geo.nDetecV*geo.nDetecU*sizeof(float), hipMemcpyDeviceToHost,stream[1]); + + + + hipDestroyTextureObject(texImg[0]); + hipFreeArray(d_cuArrTex[0]); + free(texImg); texImg = 0; + free(d_cuArrTex); d_cuArrTex = 0; + hipCheckErrors("Unbind fail"); + hipFree(dProjection[0]); + hipFree(dProjection[1]); + free(dProjection); + hipFreeHost(projParamsArrayHost); + hipCheckErrors("hipFree d_imagedata fail"); + + + for (int i = 0; i < 2; ++i){ + hipStreamDestroy(stream[i]); + } +// hipDeviceReset(); + return 0; +} + + + +/* This code precomputes The location of the source and the Delta U and delta V (in the warped space) + * to compute the locations of the x-rays. While it seems verbose and overly-optimized, + * it does saves about 30% of each of the kernel calls. Thats something! + **/ +void computeDeltas_Siddon_parallel(Geometry geo, float angles,int i, Point3D* uvorigin, Point3D* deltaU, Point3D* deltaV, Point3D* source){ + Point3D S; + + S.x =geo.DSO[i]; S.y = geo.dDetecU*(0-((float)geo.nDetecU/2)+0.5); S.z = geo.dDetecV*(((float)geo.nDetecV/2)-0.5-0); + + //End point + Point3D P,Pu0,Pv0; + + P.x =-(geo.DSD[i]-geo.DSO[i]); P.y = geo.dDetecU*(0-((float)geo.nDetecU/2)+0.5); P.z = geo.dDetecV*(((float)geo.nDetecV/2)-0.5-0); + Pu0.x=-(geo.DSD[i]-geo.DSO[i]); Pu0.y= geo.dDetecU*(1-((float)geo.nDetecU/2)+0.5); Pu0.z= geo.dDetecV*(((float)geo.nDetecV/2)-0.5-0); + Pv0.x=-(geo.DSD[i]-geo.DSO[i]); Pv0.y= geo.dDetecU*(0-((float)geo.nDetecU/2)+0.5); Pv0.z= geo.dDetecV*(((float)geo.nDetecV/2)-0.5-1); + // Geometric trasnformations: + P.x=0;Pu0.x=0;Pv0.x=0; + + // Roll pitch yaw + rollPitchYaw(geo,i,&P); + rollPitchYaw(geo,i,&Pu0); + rollPitchYaw(geo,i,&Pv0); + //Now lets translate the points where they should be: + P.x=P.x-(geo.DSD[i]-geo.DSO[i]); + Pu0.x=Pu0.x-(geo.DSD[i]-geo.DSO[i]); + Pv0.x=Pv0.x-(geo.DSD[i]-geo.DSO[i]); + + S.x=0; + // Roll pitch yaw + rollPitchYaw(geo,i,&S); + //Now lets translate the points where they should be: + S.x=S.x+geo.DSO[i]; + + //1: Offset detector + + //P.x + P.y =P.y +geo.offDetecU[i]; P.z =P.z +geo.offDetecV[i]; + Pu0.y=Pu0.y+geo.offDetecU[i]; Pu0.z=Pu0.z+geo.offDetecV[i]; + Pv0.y=Pv0.y+geo.offDetecU[i]; Pv0.z=Pv0.z+geo.offDetecV[i]; + //S doesnt need to chagne + + + //3: Rotate (around z)! + Point3D Pfinal, Pfinalu0, Pfinalv0; + + Pfinal.x =P.x*cos(geo.alpha)-P.y*sin(geo.alpha); Pfinal.y =P.y*cos(geo.alpha)+P.x*sin(geo.alpha); Pfinal.z =P.z; + Pfinalu0.x=Pu0.x*cos(geo.alpha)-Pu0.y*sin(geo.alpha); Pfinalu0.y=Pu0.y*cos(geo.alpha)+Pu0.x*sin(geo.alpha); Pfinalu0.z=Pu0.z; + Pfinalv0.x=Pv0.x*cos(geo.alpha)-Pv0.y*sin(geo.alpha); Pfinalv0.y=Pv0.y*cos(geo.alpha)+Pv0.x*sin(geo.alpha); Pfinalv0.z=Pv0.z; + + Point3D S2; + S2.x=S.x*cos(geo.alpha)-S.y*sin(geo.alpha); + S2.y=S.y*cos(geo.alpha)+S.x*sin(geo.alpha); + S2.z=S.z; + + //2: Offset image (instead of offseting image, -offset everything else) + + Pfinal.x =Pfinal.x-geo.offOrigX[i]; Pfinal.y =Pfinal.y-geo.offOrigY[i]; Pfinal.z =Pfinal.z-geo.offOrigZ[i]; + Pfinalu0.x=Pfinalu0.x-geo.offOrigX[i]; Pfinalu0.y=Pfinalu0.y-geo.offOrigY[i]; Pfinalu0.z=Pfinalu0.z-geo.offOrigZ[i]; + Pfinalv0.x=Pfinalv0.x-geo.offOrigX[i]; Pfinalv0.y=Pfinalv0.y-geo.offOrigY[i]; Pfinalv0.z=Pfinalv0.z-geo.offOrigZ[i]; + S2.x=S2.x-geo.offOrigX[i]; S2.y=S2.y-geo.offOrigY[i]; S2.z=S2.z-geo.offOrigZ[i]; + + // As we want the (0,0,0) to be in a corner of the image, we need to translate everything (after rotation); + Pfinal.x =Pfinal.x+geo.sVoxelX/2; Pfinal.y =Pfinal.y+geo.sVoxelY/2; Pfinal.z =Pfinal.z +geo.sVoxelZ/2; + Pfinalu0.x=Pfinalu0.x+geo.sVoxelX/2; Pfinalu0.y=Pfinalu0.y+geo.sVoxelY/2; Pfinalu0.z=Pfinalu0.z+geo.sVoxelZ/2; + Pfinalv0.x=Pfinalv0.x+geo.sVoxelX/2; Pfinalv0.y=Pfinalv0.y+geo.sVoxelY/2; Pfinalv0.z=Pfinalv0.z+geo.sVoxelZ/2; + S2.x =S2.x+geo.sVoxelX/2; S2.y =S2.y+geo.sVoxelY/2; S2.z =S2.z +geo.sVoxelZ/2; + + //4. Scale everything so dVoxel==1 + Pfinal.x =Pfinal.x/geo.dVoxelX; Pfinal.y =Pfinal.y/geo.dVoxelY; Pfinal.z =Pfinal.z/geo.dVoxelZ; + Pfinalu0.x=Pfinalu0.x/geo.dVoxelX; Pfinalu0.y=Pfinalu0.y/geo.dVoxelY; Pfinalu0.z=Pfinalu0.z/geo.dVoxelZ; + Pfinalv0.x=Pfinalv0.x/geo.dVoxelX; Pfinalv0.y=Pfinalv0.y/geo.dVoxelY; Pfinalv0.z=Pfinalv0.z/geo.dVoxelZ; + S2.x =S2.x/geo.dVoxelX; S2.y =S2.y/geo.dVoxelY; S2.z =S2.z/geo.dVoxelZ; + + + + //5. apply COR. Wherever everything was, now its offesetd by a bit + float CORx, CORy; + CORx=-geo.COR[i]*sin(geo.alpha)/geo.dVoxelX; + CORy= geo.COR[i]*cos(geo.alpha)/geo.dVoxelY; + Pfinal.x+=CORx; Pfinal.y+=CORy; + Pfinalu0.x+=CORx; Pfinalu0.y+=CORy; + Pfinalv0.x+=CORx; Pfinalv0.y+=CORy; + S2.x+=CORx; S2.y+=CORy; + + // return + + *uvorigin=Pfinal; + + deltaU->x=Pfinalu0.x-Pfinal.x; + deltaU->y=Pfinalu0.y-Pfinal.y; + deltaU->z=Pfinalu0.z-Pfinal.z; + + deltaV->x=Pfinalv0.x-Pfinal.x; + deltaV->y=Pfinalv0.y-Pfinal.y; + deltaV->z=Pfinalv0.z-Pfinal.z; + + *source=S2; +} +void CreateTextureParallel(float* image,Geometry geo,hipArray** d_cuArrTex, hipTextureObject_t *texImage,hipStream_t* stream){ //size_t size_image=geo.nVoxelX*geo.nVoxelY*geo.nVoxelZ; + + + const hipExtent extent = make_cudaExtent(geo.nVoxelX, geo.nVoxelY, geo.nVoxelZ); + + //hipArray Descriptor + hipChannelFormatDesc channelDesc = hipCreateChannelDesc(); + //cuda Array + hipMalloc3DArray(&d_cuArrTex[0], &channelDesc, extent); + + + hipMemcpy3DParms copyParams = {0}; + //Array creation + copyParams.srcPtr = make_cudaPitchedPtr((void *)image, extent.width*sizeof(float), extent.width, extent.height); + copyParams.dstArray = d_cuArrTex[0]; + copyParams.extent = extent; + copyParams.kind = hipMemcpyHostToDevice; + hipMemcpy3DAsync(©Params,stream[1]); + + + //Array creation End + + hipResourceDesc texRes; + memset(&texRes, 0, sizeof(hipResourceDesc)); + texRes.resType = hipResourceTypeArray; + texRes.res.array.array = d_cuArrTex[0]; + hipTextureDesc texDescr; + memset(&texDescr, 0, sizeof(hipTextureDesc)); + texDescr.normalizedCoords = false; + texDescr.filterMode = hipFilterModePoint; + texDescr.addressMode[0] = hipAddressModeBorder; + texDescr.addressMode[1] = hipAddressModeBorder; + texDescr.addressMode[2] = hipAddressModeBorder; + texDescr.readMode = hipReadModeElementType; + hipCreateTextureObject(&texImage[0], &texRes, &texDescr, NULL); + +} + +#ifndef PROJECTION_HPP + +float maxDistanceCubeXY(Geometry geo, float alpha,int i){ + /////////// + // Compute initial "t" so we access safely as less as out of bounds as possible. + ////////// + + + float maxCubX,maxCubY; + // Forgetting Z, compute max distance: diagonal+offset + maxCubX=(geo.sVoxelX/2+ abs(geo.offOrigX[i]))/geo.dVoxelX; + maxCubY=(geo.sVoxelY/2+ abs(geo.offOrigY[i]))/geo.dVoxelY; + + return geo.DSO[i]/geo.dVoxelX-sqrt(maxCubX*maxCubX+maxCubY*maxCubY); + +} + +#endif diff --git a/Common/CUDA/Siddon_projection_parallel.hpp b/Common/CUDA/Siddon_projection_parallel.hpp index c9c6fc77..570481c9 100644 --- a/Common/CUDA/Siddon_projection_parallel.hpp +++ b/Common/CUDA/Siddon_projection_parallel.hpp @@ -1,65 +1,65 @@ -/*------------------------------------------------------------------------- - * - * Header CUDA functions for ray-voxel intersection based projection - * - * - * CODE by Ander Biguri - * ---------------------------------------------------------------------------- ---------------------------------------------------------------------------- -Copyright (c) 2015, University of Bath and CERN- European Organization for -Nuclear Research -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, -this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors -may be used to endorse or promote products derived from this software without -specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------- - -Contact: tigre.toolbox@gmail.com -Codes : https://github.com/CERN/TIGRE ---------------------------------------------------------------------------- - */ - - - - - -#include "ray_interpolated_projection.hpp" -#include "types_TIGRE.hpp" -#include "GpuIds.hpp" - -#ifndef PROJECTION_PARALLEL_HPP_SIDDON -#define PROJECTION_PARALLEL_HPP_SIDDON -int siddon_ray_projection_parallel(float * img, Geometry geo, float** result,float const * const alphas,int nalpha, const GpuIds& gpuids); - -//double computeMaxLength(Geometry geo, double alpha); -void computeDeltas_Siddon_parallel(Geometry geo, float alpha,int i, Point3D* uvorigin, Point3D* deltaU, Point3D* deltaV, Point3D* source); - -//double maxDistanceCubeXY(Geometry geo, double alpha,int i); - -// below, not used -//Geometry nomralizeGeometryImage(Geometry geo); +/*------------------------------------------------------------------------- + * + * Header CUDA functions for ray-voxel intersection based projection + * + * + * CODE by Ander Biguri + * +--------------------------------------------------------------------------- +--------------------------------------------------------------------------- +Copyright (c) 2015, University of Bath and CERN- European Organization for +Nuclear Research +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + --------------------------------------------------------------------------- + +Contact: tigre.toolbox@gmail.com +Codes : https://github.com/CERN/TIGRE +--------------------------------------------------------------------------- + */ + + + + + +#include "ray_interpolated_projection.hpp" +#include "types_TIGRE.hpp" +#include "GpuIds.hpp" + +#ifndef PROJECTION_PARALLEL_HPP_SIDDON +#define PROJECTION_PARALLEL_HPP_SIDDON +int siddon_ray_projection_parallel(float * img, Geometry geo, float** result,float const * const alphas,int nalpha, const GpuIds& gpuids); + +//double computeMaxLength(Geometry geo, double alpha); +void computeDeltas_Siddon_parallel(Geometry geo, float alpha,int i, Point3D* uvorigin, Point3D* deltaU, Point3D* deltaV, Point3D* source); + +//double maxDistanceCubeXY(Geometry geo, double alpha,int i); + +// below, not used +//Geometry nomralizeGeometryImage(Geometry geo); #endif \ No newline at end of file diff --git a/Common/CUDA/TIGRE_common.cpp b/Common/CUDA/TIGRE_common.cpp index cf98e4b9..3f28ea2a 100644 --- a/Common/CUDA/TIGRE_common.cpp +++ b/Common/CUDA/TIGRE_common.cpp @@ -1,20 +1,20 @@ -#if defined(IS_FOR_PYTIGRE) -#include -#include -#include -#include "TIGRE_common.hpp" -void mexPrintf(const char* format, ...) { - PRINT_HERE(""); - va_list argpointer; - va_start(argpointer, format); - vprintf(format, argpointer); - va_end(argpointer); -} -void mexErrMsgIdAndTxt(const char* pcTag, const char* pcMsg) { - PRINT_HERE("%s %s\n", pcTag, pcMsg); - exit(1); -} -void mexWarnMsgIdAndTxt(const char* pcTag, const char* pcMsg) { - PRINT_HERE("%s %s\n", pcTag, pcMsg); -} -#endif // IS_FOR_PYTIGRE +#if defined(IS_FOR_PYTIGRE) +#include +#include +#include +#include "TIGRE_common.hpp" +void mexPrintf(const char* format, ...) { + PRINT_HERE(""); + va_list argpointer; + va_start(argpointer, format); + vprintf(format, argpointer); + va_end(argpointer); +} +void mexErrMsgIdAndTxt(const char* pcTag, const char* pcMsg) { + PRINT_HERE("%s %s\n", pcTag, pcMsg); + exit(1); +} +void mexWarnMsgIdAndTxt(const char* pcTag, const char* pcMsg) { + PRINT_HERE("%s %s\n", pcTag, pcMsg); +} +#endif // IS_FOR_PYTIGRE diff --git a/Common/CUDA/TIGRE_common.hpp b/Common/CUDA/TIGRE_common.hpp index faf8d7ab..928650cb 100644 --- a/Common/CUDA/TIGRE_common.hpp +++ b/Common/CUDA/TIGRE_common.hpp @@ -1,24 +1,24 @@ -#ifndef _COMMON_HPP_20201017_ -#define _COMMON_HPP_20201017_ - -#define STRINGIFY(n) #n -#define TOSTRING(n) STRINGIFY(n) -#define __HERE__ __FILE__ " (" TOSTRING(__LINE__) "): " -#define PRINT_HERE printf(__HERE__);printf -// #define PRINT_HERE (void*)0 - -#if defined(IS_FOR_PYTIGRE) -#ifndef IS_FOR_MATLAB_TIGRE - #define IS_FOR_MATLAB_TIGRE 0 -#endif // IS_FOR_MATLAB_TIGRE -void mexPrintf(const char*, ...); -void mexErrMsgIdAndTxt(const char* pcTag, const char* pcMsg); -void mexWarnMsgIdAndTxt(const char* pcTag, const char* pcMsg); -#else -#ifndef IS_FOR_MATLAB_TIGRE - #define IS_FOR_MATLAB_TIGRE 1 -#endif // IS_FOR_MATLAB_TIGRE -#include "mex.h" -#include "tmwtypes.h" -#endif // IS_TIGRE_FOR_PYTHON -#endif // _COMMON_HPP_20201017_ +#ifndef _COMMON_HPP_20201017_ +#define _COMMON_HPP_20201017_ + +#define STRINGIFY(n) #n +#define TOSTRING(n) STRINGIFY(n) +#define __HERE__ __FILE__ " (" TOSTRING(__LINE__) "): " +#define PRINT_HERE printf(__HERE__);printf +// #define PRINT_HERE (void*)0 + +#if defined(IS_FOR_PYTIGRE) +#ifndef IS_FOR_MATLAB_TIGRE + #define IS_FOR_MATLAB_TIGRE 0 +#endif // IS_FOR_MATLAB_TIGRE +void mexPrintf(const char*, ...); +void mexErrMsgIdAndTxt(const char* pcTag, const char* pcMsg); +void mexWarnMsgIdAndTxt(const char* pcTag, const char* pcMsg); +#else +#ifndef IS_FOR_MATLAB_TIGRE + #define IS_FOR_MATLAB_TIGRE 1 +#endif // IS_FOR_MATLAB_TIGRE +#include "mex.h" +#include "tmwtypes.h" +#endif // IS_TIGRE_FOR_PYTHON +#endif // _COMMON_HPP_20201017_ diff --git a/Common/CUDA/gpuUtils.cu b/Common/CUDA/gpuUtils.cu index 8f2754e4..7b75e403 100644 --- a/Common/CUDA/gpuUtils.cu +++ b/Common/CUDA/gpuUtils.cu @@ -1,70 +1,70 @@ - -#include "gpuUtils.hpp" -#include -#include -#include -#include - -int GetGpuIdArray(const char* kacGPUName, int* piDeviceIds, int iIdCountMax, char* pcMessage) { - if (pcMessage) { - for (int iI = 0; iI < 65535; ++iI) { - pcMessage[iI] = '\0'; - } - } - if (piDeviceIds == 0 || iIdCountMax == 0) { - return 0; - } - int iMessagePos = 0; - // Count installed GPUs. - int iCudaDeviceCount = GetGpuCount(); - iMessagePos += sprintf(pcMessage + iMessagePos, "Found GPUs: %d\n", iCudaDeviceCount); - if (iCudaDeviceCount == 0) { - // printf("No GPU found\n"); - return 0; - } - - iCudaDeviceCount = min(iCudaDeviceCount, iIdCountMax); - iMessagePos += sprintf(pcMessage + iMessagePos, "Max GPUs: %d\n", iCudaDeviceCount); - if (strlen(kacGPUName) == 0) { - // Semi-compatible mode: - // Return all GPUs - for (int iI = 0; iI < iCudaDeviceCount; ++iI) { - piDeviceIds[iI] = iI; - } - return iCudaDeviceCount; - } - - cudaError_t err; - cudaDeviceProp propDevice; - int nMatch = 0; - for (int iId = 0; iId < iCudaDeviceCount; ++iId) { - err = cudaGetDeviceProperties(&propDevice, iId); - iMessagePos += sprintf(pcMessage + iMessagePos, "propDevice.name = %s\n", propDevice.name); - if (strcmp(propDevice.name, kacGPUName) == 0) { - piDeviceIds[nMatch] = iId; - ++nMatch; - } - } - - for (int iI = 0; iI < nMatch; ++iI) { - iMessagePos += sprintf(pcMessage + iMessagePos, "%d, ", piDeviceIds[iI]); - } - return nMatch; - -} - -void GetGpuName(int iDeviceId, char* pcName) { - memset(pcName, 0, 128); - cudaError_t err; - cudaDeviceProp propDevice; - int id = iDeviceId; - err = cudaGetDeviceProperties(&propDevice, id); - memcpy(pcName, propDevice.name, strlen(propDevice.name)*sizeof(char)); -} - - -int GetGpuCount() { - int iCudaDeviceCount = 0; - cudaGetDeviceCount(&iCudaDeviceCount); - return iCudaDeviceCount; -} + +#include "gpuUtils.hpp" +#include +#include +#include +#include + +int GetGpuIdArray(const char* kacGPUName, int* piDeviceIds, int iIdCountMax, char* pcMessage) { + if (pcMessage) { + for (int iI = 0; iI < 65535; ++iI) { + pcMessage[iI] = '\0'; + } + } + if (piDeviceIds == 0 || iIdCountMax == 0) { + return 0; + } + int iMessagePos = 0; + // Count installed GPUs. + int iCudaDeviceCount = GetGpuCount(); + iMessagePos += sprintf(pcMessage + iMessagePos, "Found GPUs: %d\n", iCudaDeviceCount); + if (iCudaDeviceCount == 0) { + // printf("No GPU found\n"); + return 0; + } + + iCudaDeviceCount = min(iCudaDeviceCount, iIdCountMax); + iMessagePos += sprintf(pcMessage + iMessagePos, "Max GPUs: %d\n", iCudaDeviceCount); + if (strlen(kacGPUName) == 0) { + // Semi-compatible mode: + // Return all GPUs + for (int iI = 0; iI < iCudaDeviceCount; ++iI) { + piDeviceIds[iI] = iI; + } + return iCudaDeviceCount; + } + + hipError_t err; + hipDeviceProp propDevice; + int nMatch = 0; + for (int iId = 0; iId < iCudaDeviceCount; ++iId) { + err = hipGetDeviceProperties(&propDevice, iId); + iMessagePos += sprintf(pcMessage + iMessagePos, "propDevice.name = %s\n", propDevice.name); + if (strcmp(propDevice.name, kacGPUName) == 0) { + piDeviceIds[nMatch] = iId; + ++nMatch; + } + } + + for (int iI = 0; iI < nMatch; ++iI) { + iMessagePos += sprintf(pcMessage + iMessagePos, "%d, ", piDeviceIds[iI]); + } + return nMatch; + +} + +void GetGpuName(int iDeviceId, char* pcName) { + memset(pcName, 0, 128); + hipError_t err; + hipDeviceProp propDevice; + int id = iDeviceId; + err = hipGetDeviceProperties(&propDevice, id); + memcpy(pcName, propDevice.name, strlen(propDevice.name)*sizeof(char)); +} + + +int GetGpuCount() { + int iCudaDeviceCount = 0; + hipGetDeviceCount(&iCudaDeviceCount); + return iCudaDeviceCount; +} diff --git a/Common/CUDA/gpuUtils.hpp b/Common/CUDA/gpuUtils.hpp index 38b518cf..9cdb82fb 100644 --- a/Common/CUDA/gpuUtils.hpp +++ b/Common/CUDA/gpuUtils.hpp @@ -1,18 +1,18 @@ - -#ifndef GPUUTILS_HPP -#define GPUUTILS_HPP -//! @brief # of installed GPUs -int GetGpuCount(); - -//! @brief IDs of GPUs whose name is kacGPUName. -//! @note Call GetGpuCount and allocate sufficient memory for piDeviceIds. -//! @param [in] kacGPUName -//! @param [in, out] piDeviceIds. -//! @param [in] iIdCountMax. Return value of GetGpuCount() -int GetGpuIdArray(const char* kacGPUName, int* piDeviceIds, int iIdCountMax, char* pcMessage); - -//! @brief GPU name of index iDeviceId. Allocate 128bytes for pcName before call. -void GetGpuName(int iDeviceId, char* pcName); - -#endif // GPUUTILS_HPP - + +#ifndef GPUUTILS_HPP +#define GPUUTILS_HPP +//! @brief # of installed GPUs +int GetGpuCount(); + +//! @brief IDs of GPUs whose name is kacGPUName. +//! @note Call GetGpuCount and allocate sufficient memory for piDeviceIds. +//! @param [in] kacGPUName +//! @param [in, out] piDeviceIds. +//! @param [in] iIdCountMax. Return value of GetGpuCount() +int GetGpuIdArray(const char* kacGPUName, int* piDeviceIds, int iIdCountMax, char* pcMessage); + +//! @brief GPU name of index iDeviceId. Allocate 128bytes for pcName before call. +void GetGpuName(int iDeviceId, char* pcName); + +#endif // GPUUTILS_HPP + diff --git a/Common/CUDA/improvedForwardProjections.cu b/Common/CUDA/improvedForwardProjections.cu index 0f32be72..0f72a305 100644 --- a/Common/CUDA/improvedForwardProjections.cu +++ b/Common/CUDA/improvedForwardProjections.cu @@ -20,19 +20,19 @@ Coded by: Stefanie Kaser, Benjamin Kirchmayer --------------------------------------------------------------------------*/ -#include +#include #include "mex.h" -#include +#include #include "improvedForwardProjections.hpp" #include #include -#define cudaCheckErrors(msg) \ +#define hipCheckErrors(msg) \ do { \ - cudaError_t __err = cudaGetLastError(); \ - if (__err != cudaSuccess) { \ + hipError_t __err = hipGetLastError(); \ + if (__err != hipSuccess) { \ mexPrintf("%s \n",msg);\ - mexErrMsgIdAndTxt("ImprovedForwardProj:",cudaGetErrorString(__err));\ + mexErrMsgIdAndTxt("ImprovedForwardProj:",hipGetErrorString(__err));\ } \ } while (0) @@ -937,44 +937,44 @@ __host__ void ParticleProjections(float * outProjection, float* posIn, float* po } //Allocate Memory on GPU - cudaMalloc( (void**) &dPosIn, sizeInputs ); - cudaMalloc( (void**) &dPosOut, sizeInputs ); - cudaMalloc( (void**) &ddirIn, sizeInputs ); - cudaMalloc( (void**) &ddirOut, sizeInputs ); - cudaMalloc( (void**) &d_wepl, numOfEntries*sizeof(float)); - cudaMalloc( (void**) &dhist1, detectorMem ); - cudaMalloc( (void**) &dhist2, detectorMem ); - cudaMalloc( (void**) &dnumEntries, sizeof(int)); - cudaMalloc( (void**) &ddetectorX, sizeof(int)); - cudaMalloc( (void**) &ddetectorY, sizeof(int)); - cudaMalloc( (void**) &dpixelSize, 2*sizeof(float)); - cudaMalloc( (void**) &dDetectDistIn, sizeof(float)); - cudaMalloc( (void**) &dDetectDistOut, sizeof(float)); - cudaMalloc( (void**) &dEin, sizeof(float)); - cudaMalloc( (void**) &dReject, sizeof(float)); - cudaMalloc( (void**) &dHull, 5*sizeof(float)); - cudaError_t _err_alloc = cudaGetLastError(); - mexPrintf("%s \n", cudaGetErrorString(_err_alloc)); - cudaCheckErrors("GPU Allocation failed!"); + hipMalloc( (void**) &dPosIn, sizeInputs ); + hipMalloc( (void**) &dPosOut, sizeInputs ); + hipMalloc( (void**) &ddirIn, sizeInputs ); + hipMalloc( (void**) &ddirOut, sizeInputs ); + hipMalloc( (void**) &d_wepl, numOfEntries*sizeof(float)); + hipMalloc( (void**) &dhist1, detectorMem ); + hipMalloc( (void**) &dhist2, detectorMem ); + hipMalloc( (void**) &dnumEntries, sizeof(int)); + hipMalloc( (void**) &ddetectorX, sizeof(int)); + hipMalloc( (void**) &ddetectorY, sizeof(int)); + hipMalloc( (void**) &dpixelSize, 2*sizeof(float)); + hipMalloc( (void**) &dDetectDistIn, sizeof(float)); + hipMalloc( (void**) &dDetectDistOut, sizeof(float)); + hipMalloc( (void**) &dEin, sizeof(float)); + hipMalloc( (void**) &dReject, sizeof(float)); + hipMalloc( (void**) &dHull, 5*sizeof(float)); + hipError_t _err_alloc = hipGetLastError(); + mexPrintf("%s \n", hipGetErrorString(_err_alloc)); + hipCheckErrors("GPU Allocation failed!"); //Copy Arrays to GPU - cudaMemcpy(dPosIn, posIn,sizeInputs ,cudaMemcpyHostToDevice); - cudaMemcpy(dPosOut, posOut,sizeInputs,cudaMemcpyHostToDevice); - cudaMemcpy(ddirIn, dirIn,sizeInputs,cudaMemcpyHostToDevice); - cudaMemcpy(ddirOut, dirOut,sizeInputs,cudaMemcpyHostToDevice); - cudaMemcpy(d_wepl, p_wepl, numOfEntries*sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dnumEntries, &numOfEntries,sizeof(int), cudaMemcpyHostToDevice); - cudaMemcpy(ddetectorX, &detectSizeX, sizeof(int), cudaMemcpyHostToDevice); - cudaMemcpy(ddetectorY, &detectSizeY, sizeof(int), cudaMemcpyHostToDevice); - cudaMemcpy(dpixelSize, pixelSize, 2*sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dDetectDistIn, &detectDistIn, sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dDetectDistOut, &detectDistOut, sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dEin, &ein, sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dReject, &reject, sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dHull, ch_param, 5*sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dhist1, hist1, detectorMem, cudaMemcpyHostToDevice); - cudaMemcpy(dhist2, hist2, detectorMem, cudaMemcpyHostToDevice); - cudaCheckErrors("Host to device transport failed!"); + hipMemcpy(dPosIn, posIn,sizeInputs ,hipMemcpyHostToDevice); + hipMemcpy(dPosOut, posOut,sizeInputs,hipMemcpyHostToDevice); + hipMemcpy(ddirIn, dirIn,sizeInputs,hipMemcpyHostToDevice); + hipMemcpy(ddirOut, dirOut,sizeInputs,hipMemcpyHostToDevice); + hipMemcpy(d_wepl, p_wepl, numOfEntries*sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dnumEntries, &numOfEntries,sizeof(int), hipMemcpyHostToDevice); + hipMemcpy(ddetectorX, &detectSizeX, sizeof(int), hipMemcpyHostToDevice); + hipMemcpy(ddetectorY, &detectSizeY, sizeof(int), hipMemcpyHostToDevice); + hipMemcpy(dpixelSize, pixelSize, 2*sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dDetectDistIn, &detectDistIn, sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dDetectDistOut, &detectDistOut, sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dEin, &ein, sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dReject, &reject, sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dHull, ch_param, 5*sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dhist1, hist1, detectorMem, hipMemcpyHostToDevice); + hipMemcpy(dhist2, hist2, detectorMem, hipMemcpyHostToDevice); + hipCheckErrors("Host to device transport failed!"); @@ -984,22 +984,22 @@ __host__ void ParticleProjections(float * outProjection, float* posIn, float* po ParticleKernel<<>>(dhist1, dhist2, dPosIn, dPosOut, ddirIn, ddirOut, d_wepl, dnumEntries, ddetectorX, ddetectorY, \ dpixelSize, dDetectDistIn, dDetectDistOut, dEin, dHull, dReject); - cudaError_t _err = cudaGetLastError(); - mexPrintf("%s \n", cudaGetErrorString(_err)); - cudaCheckErrors("Kernel fail!"); + hipError_t _err = hipGetLastError(); + mexPrintf("%s \n", hipGetErrorString(_err)); + hipCheckErrors("Kernel fail!"); //dim3 grid_sum((int)floor(detectSizeX*detectSizeY/64),1,1); //dim3 block_sum(64,1,1); //sumHist<<>>(dhist1, dhist2); //Copy result from device to host - //cudaMemcpy(outProjection, dhist1,detectorMem ,cudaMemcpyDeviceToHost); - cudaMemcpy(hist1, dhist1,detectorMem ,cudaMemcpyDeviceToHost); - cudaMemcpy(hist2, dhist2,detectorMem ,cudaMemcpyDeviceToHost); - cudaMemcpy(&reject, dReject,sizeof(float) ,cudaMemcpyDeviceToHost); - //cudaError_t _errcp = cudaGetLastError(); - //mexPrintf("%s \n", cudaGetErrorString(_errcp)); - cudaCheckErrors("Device to host transport failed!"); + //hipMemcpy(outProjection, dhist1,detectorMem ,hipMemcpyDeviceToHost); + hipMemcpy(hist1, dhist1,detectorMem ,hipMemcpyDeviceToHost); + hipMemcpy(hist2, dhist2,detectorMem ,hipMemcpyDeviceToHost); + hipMemcpy(&reject, dReject,sizeof(float) ,hipMemcpyDeviceToHost); + //hipError_t _errcp = hipGetLastError(); + //mexPrintf("%s \n", hipGetErrorString(_errcp)); + hipCheckErrors("Device to host transport failed!"); for(int j = 0; j -#include +#include +#include #include #ifndef improvedForwardProjections_H #define improvedForwardProjections_H diff --git a/Common/CUDA/improvedForwardProjections_cone.cu b/Common/CUDA/improvedForwardProjections_cone.cu index 7a4f6b46..d9861b26 100644 --- a/Common/CUDA/improvedForwardProjections_cone.cu +++ b/Common/CUDA/improvedForwardProjections_cone.cu @@ -21,19 +21,19 @@ --------------------------------------------------------------------------*/ -#include +#include #include "mex.h" -#include +#include #include "improvedForwardProjections.hpp" // #include // #include -#define cudaCheckErrors(msg) \ +#define hipCheckErrors(msg) \ do { \ - cudaError_t __err = cudaGetLastError(); \ - if (__err != cudaSuccess) { \ + hipError_t __err = hipGetLastError(); \ + if (__err != hipSuccess) { \ mexPrintf("%s \n",msg);\ - mexErrMsgIdAndTxt("ImprovedForwardProj:",cudaGetErrorString(__err));\ + mexErrMsgIdAndTxt("ImprovedForwardProj:",hipGetErrorString(__err));\ } \ } while (0) @@ -1133,46 +1133,46 @@ __host__ void ParticleProjectionsCone(float * outProjection, float* posIn, float } //Allocate Memory on GPU - cudaMalloc( (void**) &dPosIn, sizeInputs ); - cudaMalloc( (void**) &dPosOut, sizeInputs ); - cudaMalloc( (void**) &ddirIn, sizeInputs ); - cudaMalloc( (void**) &ddirOut, sizeInputs ); - cudaMalloc( (void**) &d_wepl, numOfEntries*sizeof(float)); - cudaMalloc( (void**) &dhist1, detectorMem ); - cudaMalloc( (void**) &dhist2, detectorMem ); - cudaMalloc( (void**) &dnumEntries, sizeof(int)); - cudaMalloc( (void**) &ddetectorX, sizeof(int)); - cudaMalloc( (void**) &ddetectorY, sizeof(int)); - cudaMalloc( (void**) &dpixelSize, 2*sizeof(float)); - cudaMalloc( (void**) &dDetectDistIn, sizeof(float)); - cudaMalloc( (void**) &dDetectDistOut, sizeof(float)); - cudaMalloc( (void**) &dSourceDist, sizeof(float)); - cudaMalloc( (void**) &dEin, sizeof(float)); - cudaMalloc( (void**) &dReject, sizeof(float)); - cudaMalloc( (void**) &dHull, 5*sizeof(float)); - cudaError_t _err_alloc = cudaGetLastError(); - mexPrintf("%s \n", cudaGetErrorString(_err_alloc)); - cudaCheckErrors("GPU Allocation failed!"); + hipMalloc( (void**) &dPosIn, sizeInputs ); + hipMalloc( (void**) &dPosOut, sizeInputs ); + hipMalloc( (void**) &ddirIn, sizeInputs ); + hipMalloc( (void**) &ddirOut, sizeInputs ); + hipMalloc( (void**) &d_wepl, numOfEntries*sizeof(float)); + hipMalloc( (void**) &dhist1, detectorMem ); + hipMalloc( (void**) &dhist2, detectorMem ); + hipMalloc( (void**) &dnumEntries, sizeof(int)); + hipMalloc( (void**) &ddetectorX, sizeof(int)); + hipMalloc( (void**) &ddetectorY, sizeof(int)); + hipMalloc( (void**) &dpixelSize, 2*sizeof(float)); + hipMalloc( (void**) &dDetectDistIn, sizeof(float)); + hipMalloc( (void**) &dDetectDistOut, sizeof(float)); + hipMalloc( (void**) &dSourceDist, sizeof(float)); + hipMalloc( (void**) &dEin, sizeof(float)); + hipMalloc( (void**) &dReject, sizeof(float)); + hipMalloc( (void**) &dHull, 5*sizeof(float)); + hipError_t _err_alloc = hipGetLastError(); + mexPrintf("%s \n", hipGetErrorString(_err_alloc)); + hipCheckErrors("GPU Allocation failed!"); //Copy Arrays to GPU - cudaMemcpy(dPosIn, posIn,sizeInputs ,cudaMemcpyHostToDevice); - cudaMemcpy(dPosOut, posOut,sizeInputs,cudaMemcpyHostToDevice); - cudaMemcpy(ddirIn, dirIn,sizeInputs,cudaMemcpyHostToDevice); - cudaMemcpy(ddirOut, dirOut,sizeInputs,cudaMemcpyHostToDevice); - cudaMemcpy(d_wepl, p_wepl, numOfEntries*sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dnumEntries, &numOfEntries,sizeof(int), cudaMemcpyHostToDevice); - cudaMemcpy(ddetectorX, &detectSizeX, sizeof(int), cudaMemcpyHostToDevice); - cudaMemcpy(ddetectorY, &detectSizeY, sizeof(int), cudaMemcpyHostToDevice); - cudaMemcpy(dpixelSize, pixelSize, 2*sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dDetectDistIn, &detectDistIn, sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dDetectDistOut, &detectDistOut, sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dSourceDist, &sourcePos, sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dEin, &ein, sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dReject, &reject, sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dHull, ch_param, 5*sizeof(float), cudaMemcpyHostToDevice); - cudaMemcpy(dhist1, hist1, detectorMem, cudaMemcpyHostToDevice); - cudaMemcpy(dhist2, hist2, detectorMem, cudaMemcpyHostToDevice); - cudaCheckErrors("Host to device transport failed!"); + hipMemcpy(dPosIn, posIn,sizeInputs ,hipMemcpyHostToDevice); + hipMemcpy(dPosOut, posOut,sizeInputs,hipMemcpyHostToDevice); + hipMemcpy(ddirIn, dirIn,sizeInputs,hipMemcpyHostToDevice); + hipMemcpy(ddirOut, dirOut,sizeInputs,hipMemcpyHostToDevice); + hipMemcpy(d_wepl, p_wepl, numOfEntries*sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dnumEntries, &numOfEntries,sizeof(int), hipMemcpyHostToDevice); + hipMemcpy(ddetectorX, &detectSizeX, sizeof(int), hipMemcpyHostToDevice); + hipMemcpy(ddetectorY, &detectSizeY, sizeof(int), hipMemcpyHostToDevice); + hipMemcpy(dpixelSize, pixelSize, 2*sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dDetectDistIn, &detectDistIn, sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dDetectDistOut, &detectDistOut, sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dSourceDist, &sourcePos, sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dEin, &ein, sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dReject, &reject, sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dHull, ch_param, 5*sizeof(float), hipMemcpyHostToDevice); + hipMemcpy(dhist1, hist1, detectorMem, hipMemcpyHostToDevice); + hipMemcpy(dhist2, hist2, detectorMem, hipMemcpyHostToDevice); + hipCheckErrors("Host to device transport failed!"); @@ -1182,22 +1182,22 @@ __host__ void ParticleProjectionsCone(float * outProjection, float* posIn, float ParticleKernelCone<<>>(dhist1, dhist2, dPosIn, dPosOut, ddirIn, ddirOut, d_wepl, dnumEntries, ddetectorX, ddetectorY, \ dpixelSize, dDetectDistIn, dDetectDistOut, dEin, dHull, dReject, dSourceDist); - cudaError_t _err = cudaGetLastError(); - mexPrintf("%s \n", cudaGetErrorString(_err)); - cudaCheckErrors("Kernel fail!"); + hipError_t _err = hipGetLastError(); + mexPrintf("%s \n", hipGetErrorString(_err)); + hipCheckErrors("Kernel fail!"); //dim3 grid_sum((int)floor(detectSizeX*detectSizeY/64),1,1); //dim3 block_sum(64,1,1); //sumHist<<>>(dhist1, dhist2); //Copy result from device to host - //cudaMemcpy(outProjection, dhist1,detectorMem ,cudaMemcpyDeviceToHost); - cudaMemcpy(hist1, dhist1,detectorMem ,cudaMemcpyDeviceToHost); - cudaMemcpy(hist2, dhist2,detectorMem ,cudaMemcpyDeviceToHost); - cudaMemcpy(&reject, dReject,sizeof(float) ,cudaMemcpyDeviceToHost); - //cudaError_t _errcp = cudaGetLastError(); - //mexPrintf("%s \n", cudaGetErrorString(_errcp)); - cudaCheckErrors("Device to host transport failed!"); + //hipMemcpy(outProjection, dhist1,detectorMem ,hipMemcpyDeviceToHost); + hipMemcpy(hist1, dhist1,detectorMem ,hipMemcpyDeviceToHost); + hipMemcpy(hist2, dhist2,detectorMem ,hipMemcpyDeviceToHost); + hipMemcpy(&reject, dReject,sizeof(float) ,hipMemcpyDeviceToHost); + //hipError_t _errcp = hipGetLastError(); + //mexPrintf("%s \n", hipGetErrorString(_errcp)); + hipCheckErrors("Device to host transport failed!"); for(int j = 0; j -#include - -float maxDistanceCubeXY(Geometry geo, float alpha,int i){ - /////////// - // Compute initial "t" so we access safely as less as out of bounds as possible. - ////////// - - float maxCubX,maxCubY; - // Forgetting Z, compute max distance: diagonal+offset - maxCubX=(geo.sVoxelX/2+ abs(geo.offOrigX[i]))/geo.dVoxelX; - maxCubY=(geo.sVoxelY/2+ abs(geo.offOrigY[i]))/geo.dVoxelY; - - return geo.DSO[i]/geo.dVoxelX-sqrt(maxCubX*maxCubX+maxCubY*maxCubY); -} - -void rollPitchYaw(Geometry geo,int i, Point3D* point){ - Point3D auxPoint; - auxPoint.x=point->x; - auxPoint.y=point->y; - auxPoint.z=point->z; - - point->x=cos(geo.dRoll[i])*cos(geo.dPitch[i])*auxPoint.x - +(cos(geo.dRoll[i])*sin(geo.dPitch[i])*sin(geo.dYaw[i]) - sin(geo.dRoll[i])*cos(geo.dYaw[i]))*auxPoint.y - +(cos(geo.dRoll[i])*sin(geo.dPitch[i])*cos(geo.dYaw[i]) + sin(geo.dRoll[i])*sin(geo.dYaw[i]))*auxPoint.z; - - point->y=sin(geo.dRoll[i])*cos(geo.dPitch[i])*auxPoint.x - +(sin(geo.dRoll[i])*sin(geo.dPitch[i])*sin(geo.dYaw[i]) + cos(geo.dRoll[i])*cos(geo.dYaw[i]))*auxPoint.y - +(sin(geo.dRoll[i])*sin(geo.dPitch[i])*cos(geo.dYaw[i]) - cos(geo.dRoll[i])*sin(geo.dYaw[i]))*auxPoint.z; - - point->z=-sin(geo.dPitch[i])*auxPoint.x - +cos(geo.dPitch[i])*sin(geo.dYaw[i])*auxPoint.y - +cos(geo.dPitch[i])*cos(geo.dYaw[i])*auxPoint.z; +#include "projection.hpp" +#include +#include + +float maxDistanceCubeXY(Geometry geo, float alpha,int i){ + /////////// + // Compute initial "t" so we access safely as less as out of bounds as possible. + ////////// + + float maxCubX,maxCubY; + // Forgetting Z, compute max distance: diagonal+offset + maxCubX=(geo.sVoxelX/2+ abs(geo.offOrigX[i]))/geo.dVoxelX; + maxCubY=(geo.sVoxelY/2+ abs(geo.offOrigY[i]))/geo.dVoxelY; + + return geo.DSO[i]/geo.dVoxelX-sqrt(maxCubX*maxCubX+maxCubY*maxCubY); +} + +void rollPitchYaw(Geometry geo,int i, Point3D* point){ + Point3D auxPoint; + auxPoint.x=point->x; + auxPoint.y=point->y; + auxPoint.z=point->z; + + point->x=cos(geo.dRoll[i])*cos(geo.dPitch[i])*auxPoint.x + +(cos(geo.dRoll[i])*sin(geo.dPitch[i])*sin(geo.dYaw[i]) - sin(geo.dRoll[i])*cos(geo.dYaw[i]))*auxPoint.y + +(cos(geo.dRoll[i])*sin(geo.dPitch[i])*cos(geo.dYaw[i]) + sin(geo.dRoll[i])*sin(geo.dYaw[i]))*auxPoint.z; + + point->y=sin(geo.dRoll[i])*cos(geo.dPitch[i])*auxPoint.x + +(sin(geo.dRoll[i])*sin(geo.dPitch[i])*sin(geo.dYaw[i]) + cos(geo.dRoll[i])*cos(geo.dYaw[i]))*auxPoint.y + +(sin(geo.dRoll[i])*sin(geo.dPitch[i])*cos(geo.dYaw[i]) - cos(geo.dRoll[i])*sin(geo.dYaw[i]))*auxPoint.z; + + point->z=-sin(geo.dPitch[i])*auxPoint.x + +cos(geo.dPitch[i])*sin(geo.dYaw[i])*auxPoint.y + +cos(geo.dPitch[i])*cos(geo.dYaw[i])*auxPoint.z; } \ No newline at end of file diff --git a/Common/CUDA/projection.hpp b/Common/CUDA/projection.hpp index 54597d92..36a0f6f5 100644 --- a/Common/CUDA/projection.hpp +++ b/Common/CUDA/projection.hpp @@ -1,9 +1,9 @@ -#ifndef PROJECTION_HPP -#define PROJECTION_HPP - -#include "types_TIGRE.hpp" - -float maxDistanceCubeXY(Geometry geo, float alpha,int i); -void rollPitchYaw(Geometry geo,int i, Point3D* point); - -#endif +#ifndef PROJECTION_HPP +#define PROJECTION_HPP + +#include "types_TIGRE.hpp" + +float maxDistanceCubeXY(Geometry geo, float alpha,int i); +void rollPitchYaw(Geometry geo,int i, Point3D* point); + +#endif diff --git a/Common/CUDA/ray_interpolated_projection.cu b/Common/CUDA/ray_interpolated_projection.cu index e71c5b59..473d8e91 100644 --- a/Common/CUDA/ray_interpolated_projection.cu +++ b/Common/CUDA/ray_interpolated_projection.cu @@ -1,843 +1,843 @@ -/*------------------------------------------------------------------------- - * - * CUDA functions for texture-memory interpolation based projection - * - * This file has the necessary fucntiosn to perform X-ray CBCT projection - * operation given a geaometry, angles and image. It uses the 3D texture - * memory linear interpolation to uniformily sample a path to integrate the - * X-rays. - * - * CODE by Ander Biguri - * Sepideh Hatamikia (arbitrary rotation) - * --------------------------------------------------------------------------- - * --------------------------------------------------------------------------- - * Copyright (c) 2015, University of Bath and CERN- European Organization for - * Nuclear Research - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * --------------------------------------------------------------------------- - * - * Contact: tigre.toolbox@gmail.com - * Codes : https://github.com/CERN/TIGRE - * --------------------------------------------------------------------------- - */ - - - - - - -#include -#include -#include -#include "ray_interpolated_projection.hpp" -#include "TIGRE_common.hpp" -#include - -#define cudaCheckErrors(msg) \ -do { \ - cudaError_t __err = cudaGetLastError(); \ - if (__err != cudaSuccess) { \ - mexPrintf("%s \n",msg);\ - cudaDeviceReset();\ - mexErrMsgIdAndTxt("TIGRE:Ax:interpolated",cudaGetErrorString(__err));\ - } \ -} while (0) - - - -#define MAXTREADS 1024 -#define PROJ_PER_BLOCK 9 -#define PIXEL_SIZE_BLOCK 9 - /*GEOMETRY DEFINITION - * - * Detector plane, behind - * |-----------------------------| - * | | - * | | - * | | - * | | - * | +--------+ | - * | / /| | - * A Z | / / |*D | - * | | +--------+ | | - * | | | | | | - * | | | *O | + | - * --->y | | | / | - * / | | |/ | - * V X | +--------+ | - * |-----------------------------| - * - * *S - * - * - * - * - * - **/ - void CreateTextureInterp(const GpuIds& gpuids,const float* imagedata,Geometry geo,cudaArray** d_cuArrTex, cudaTextureObject_t *texImage,bool allocate); -__constant__ Point3D projParamsArrayDev[4*PROJ_PER_BLOCK]; // Dev means it is on device -__constant__ float projFloatsArrayDev[2*PROJ_PER_BLOCK]; // Dev means it is on device - - -__global__ void vecAddInPlaceInterp(float *a, float *b, unsigned long n) -{ - int idx = blockIdx.x*blockDim.x+threadIdx.x; - // Make sure we do not go out of bounds - if (idx < n) - a[idx] = a[idx] + b[idx]; -} - - -template - __global__ void kernelPixelDetector( Geometry geo, - float* detector, - const int currProjSetNumber, - const int totalNoOfProjections, - cudaTextureObject_t tex){ - - unsigned long long u = blockIdx.x * blockDim.x + threadIdx.x; - unsigned long long v = blockIdx.y * blockDim.y + threadIdx.y; - unsigned long long projNumber=threadIdx.z; - - if (u>= geo.nDetecU || v>= geo.nDetecV || projNumber>=PROJ_PER_BLOCK) - return; - -#if IS_FOR_MATLAB_TIGRE - size_t idx = (size_t)(u * (unsigned long long)geo.nDetecV + v)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; -#else - size_t idx = (size_t)(v * (unsigned long long)geo.nDetecU + u)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; -#endif - - unsigned long indAlpha = currProjSetNumber*PROJ_PER_BLOCK+projNumber; // This is the ABSOLUTE projection number in the projection array - - if(indAlpha>=totalNoOfProjections) - return; - - Point3D uvOrigin = projParamsArrayDev[4*projNumber]; // 6*projNumber because we have 6 Point3D values per projection - Point3D deltaU = projParamsArrayDev[4*projNumber+1]; - Point3D deltaV = projParamsArrayDev[4*projNumber+2]; - Point3D source = projParamsArrayDev[4*projNumber+3]; - - float DSO = projFloatsArrayDev[2*projNumber+0]; - float cropdist_init = projFloatsArrayDev[2*projNumber+1]; - - - - /////// Get coordinates XYZ of pixel UV - unsigned long pixelV = geo.nDetecV-v-1; - unsigned long pixelU = u; - - - float vectX,vectY,vectZ; - Point3D P; - P.x=(uvOrigin.x+pixelU*deltaU.x+pixelV*deltaV.x); - P.y=(uvOrigin.y+pixelU*deltaU.y+pixelV*deltaV.y); - P.z=(uvOrigin.z+pixelU*deltaU.z+pixelV*deltaV.z); - - // Length is the ray length in normalized space - float length=__fsqrt_rd((source.x-P.x)*(source.x-P.x)+(source.y-P.y)*(source.y-P.y)+(source.z-P.z)*(source.z-P.z)); - //now legth is an integer of Nsamples that are required on this line - length=ceilf(__fdividef(length,geo.accuracy));//Divide the directional vector by an integer - vectX=__fdividef(P.x -source.x,length); - vectY=__fdividef(P.y -source.y,length); - vectZ=__fdividef(P.z -source.z,length); - - -// //Integrate over the line - float tx,ty,tz; - float sum=0; - float i; - - - -// Because I have no idea how to efficiently cutoff the legth path in 3D, a very upper limit is computed (see maxdistanceCuboid) -// for the 3D case. However it would be bad to lose performance in the 3D case -// TODO: can ge really improve this? - if (sphericalrotation){ - if ((2*DSO/fminf(fminf(geo.dVoxelX,geo.dVoxelY),geo.dVoxelZ)+cropdist_init)/geo.accuracy < length) - length=ceilf((2*DSO/fminf(fminf(geo.dVoxelX,geo.dVoxelY),geo.dVoxelZ)+cropdist_init)/geo.accuracy); - } - else{ - if ((2*DSO/fminf(geo.dVoxelX,geo.dVoxelY)+cropdist_init)/geo.accuracy < length) - length=ceilf((2*DSO/fminf(geo.dVoxelX,geo.dVoxelY)+cropdist_init)/geo.accuracy); - } - - - //Length is not actually a length, but the amount of memreads with given accuracy ("samples per voxel") - for (i=floorf(cropdist_init/geo.accuracy); i<=length; i=i+1){ - tx=vectX*i+source.x; - ty=vectY*i+source.y; - tz=vectZ*i+source.z; - - sum += tex3D(tex, tx+0.5f, ty+0.5f, tz+0.5f); // this line is 94% of time. - } - - float deltalength=sqrtf((vectX*geo.dVoxelX)*(vectX*geo.dVoxelX)+ - (vectY*geo.dVoxelY)*(vectY*geo.dVoxelY)+ - (vectZ*geo.dVoxelZ)*(vectZ*geo.dVoxelZ) ); - - detector[idx]=sum*deltalength; -} - - - -// legnth(angles)=3 x nagnles, as we have roll, pitch, yaw. -int interpolation_projection(float * img, Geometry geo, float** result,float const * const angles,int nangles, const GpuIds& gpuids){ - - - // Prepare for MultiGPU - int deviceCount = gpuids.GetLength(); - cudaCheckErrors("Device query fail"); - if (deviceCount == 0) { - mexErrMsgIdAndTxt("Ax:Interpolated_projection:GPUselect","There are no available device(s) that support CUDA\n"); - } - // - // CODE assumes - // 1.-All available devices are usable by this code - // 2.-All available devices are equal, they are the same machine (warning thrown) - // Check the available devices, and if they are the same - if (!gpuids.AreEqualDevices()) { - mexWarnMsgIdAndTxt("Ax:Interpolated_projection:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); - } - int dev; - - // Check free memory - size_t mem_GPU_global; - checkFreeMemory(gpuids,&mem_GPU_global); - - // printf("geo.nDetec (U, V) = %d, %d\n", geo.nDetecU, geo.nDetecV); - - size_t mem_image=(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY*(unsigned long long)geo.nVoxelZ*sizeof(float); - size_t mem_proj =(unsigned long long)geo.nDetecU*(unsigned long long)geo.nDetecV * sizeof(float); - - // Does everything fit in the GPUs? - const bool fits_in_memory = mem_image+2*PROJ_PER_BLOCK*mem_proj= 9020 - cudaDeviceGetAttribute(&isHostRegisterSupported,cudaDevAttrHostRegisterSupported,gpuids[0]); -#endif - // empirical testing shows that when the image split is smaller than 1 (also implies the image is not very big), the time to - // pin the memory is greater than the lost time in Synchronously launching the memcpys. This is only worth it when the image is too big. - -#ifndef NO_PINNED_MEMORY - if (isHostRegisterSupported & splits>1){ - cudaHostRegister(img, (size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geo.nVoxelZ*(size_t)sizeof(float),cudaHostRegisterPortable); - } - cudaCheckErrors("Error pinning memory"); -#endif - Point3D source, deltaU, deltaV, uvOrigin; - - Point3D* projParamsArrayHost = 0; - cudaMallocHost((void**)&projParamsArrayHost,4*PROJ_PER_BLOCK*sizeof(Point3D)); - float* projFloatsArrayHost = 0; - cudaMallocHost((void**)&projFloatsArrayHost,2*PROJ_PER_BLOCK*sizeof(float)); - cudaCheckErrors("Error allocating auxiliary constant memory"); - - // Create Streams for overlapping memcopy and compute - int nStream_device=2; - int nStreams=deviceCount*nStream_device; - cudaStream_t* stream=(cudaStream_t*)malloc(nStreams*sizeof(cudaStream_t)); - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - for (int i = 0; i < nStream_device; ++i){ - cudaStreamCreate(&stream[i+dev*nStream_device]); - - } - } - cudaCheckErrors("Stream creation fail"); - int nangles_device=(nangles+deviceCount-1)/deviceCount; - int nangles_last_device=(nangles-(deviceCount-1)*nangles_device); - unsigned int noOfKernelCalls = (nangles_device+PROJ_PER_BLOCK-1)/PROJ_PER_BLOCK; // We'll take care of bounds checking inside the loop if nalpha is not divisible by PROJ_PER_BLOCK - unsigned int noOfKernelCallsLastDev = (nangles_last_device+PROJ_PER_BLOCK-1)/PROJ_PER_BLOCK; // we will use this in the memory management. - int projection_this_block; - - - - cudaTextureObject_t *texImg = new cudaTextureObject_t[deviceCount]; - cudaArray **d_cuArrTex = new cudaArray*[deviceCount]; - for (unsigned int sp=0;sp=nangles) - break; - if ((i*PROJ_PER_BLOCK+j)>=nangles_device) - break; - geoArray[sp].alpha=angles[proj_global*3]; - geoArray[sp].theta=angles[proj_global*3+1]; - geoArray[sp].psi =angles[proj_global*3+2]; - - is_spherical+=abs(geoArray[sp].theta)+abs(geoArray[sp].psi); - - //precomute distances for faster execution - maxdist=maxdistanceCuboid(geoArray[sp],proj_global); - //Precompute per angle constant stuff for speed - computeDeltas(geoArray[sp], proj_global, &uvOrigin, &deltaU, &deltaV, &source); - //Ray tracing! - projParamsArrayHost[4*j]=uvOrigin; // 6*j because we have 6 Point3D values per projection - projParamsArrayHost[4*j+1]=deltaU; - projParamsArrayHost[4*j+2]=deltaV; - projParamsArrayHost[4*j+3]=source; - - projFloatsArrayHost[2*j]=geo.DSO[proj_global]; - projFloatsArrayHost[2*j+1]=floor(maxdist); - } - - cudaMemcpyToSymbolAsync(projParamsArrayDev, projParamsArrayHost, sizeof(Point3D)*4*PROJ_PER_BLOCK,0,cudaMemcpyHostToDevice,stream[dev*nStream_device]); - cudaMemcpyToSymbolAsync(projFloatsArrayDev, projFloatsArrayHost, sizeof(float)*2*PROJ_PER_BLOCK,0,cudaMemcpyHostToDevice,stream[dev*nStream_device]); - cudaStreamSynchronize(stream[dev*nStream_device]); - - - //TODO: we could do this around X and Y axis too, but we would need to compute the new axis of rotation (not possible to know from jsut the angles) - if (!is_spherical){ - kernelPixelDetector<<>>(geoArray[sp],dProjection[(i%2)+dev*2],i,nangles_device,texImg[dev]); - } - else{ - kernelPixelDetector <<>>(geoArray[sp],dProjection[(i%2)+dev*2],i,nangles_device,texImg[dev]); - } - } - - - // Now that the computation is happening, we need to either prepare the memory for - // combining of the projections (splits>1) and start removing previous results. - - - // If our image does not fit in memory then we need to make sure we accumulate previous results too. - // This is done in 2 steps: - // 1)copy previous results back into GPU - // 2)accumulate with current results - // The code to take them out is the same as when there are no splits needed - if( !fits_in_memory&&sp>0) - { - // 1) grab previous results and put them in the auxiliary variable dProjection_accum - for (dev = 0; dev < deviceCount; dev++) - { - cudaSetDevice(gpuids[dev]); - //Global index of FIRST projection on this set on this GPU - proj_global=i*PROJ_PER_BLOCK+dev*nangles_device; - if(proj_global>=nangles) - break; - - // Unless its the last projection set, we have PROJ_PER_BLOCK angles. Otherwise... - if(i+1==noOfKernelCalls) //is it the last block? - projection_this_block=min(nangles_device-(noOfKernelCalls-1)*PROJ_PER_BLOCK, //the remaining angles that this GPU had to do (almost never PROJ_PER_BLOCK) - nangles-proj_global); //or whichever amount is left to finish all (this is for the last GPU) - else - projection_this_block=PROJ_PER_BLOCK; - cudaMemcpyAsync(dProjection_accum[(i%2)+dev*2], result[proj_global], projection_this_block*geo.nDetecV*geo.nDetecU*sizeof(float), cudaMemcpyHostToDevice,stream[dev*2+1]); - } - // 2) take the results from current compute call and add it to the code in execution. - for (dev = 0; dev < deviceCount; dev++) - { - cudaSetDevice(gpuids[dev]); - //Global index of FIRST projection on this set on this GPU - proj_global=i*PROJ_PER_BLOCK+dev*nangles_device; - if(proj_global>=nangles) - break; - - // Unless its the last projection set, we have PROJ_PER_BLOCK angles. Otherwise... - if(i+1==noOfKernelCalls) //is it the last block? - projection_this_block=min(nangles_device-(noOfKernelCalls-1)*PROJ_PER_BLOCK, //the remaining angles that this GPU had to do (almost never PROJ_PER_BLOCK) - nangles-proj_global); //or whichever amount is left to finish all (this is for the last GPU) - else - projection_this_block=PROJ_PER_BLOCK; - cudaStreamSynchronize(stream[dev*2+1]); // wait until copy is finished - vecAddInPlaceInterp<<<(geo.nDetecU*geo.nDetecV*projection_this_block+MAXTREADS-1)/MAXTREADS,MAXTREADS,0,stream[dev*2]>>>(dProjection[(i%2)+dev*2],dProjection_accum[(i%2)+dev*2],(unsigned long)geo.nDetecU*geo.nDetecV*projection_this_block); - } - } // end accumulation case, where the image needs to be split - - // Now, lets get out the projections from the previous execution of the kernels. - if (i>0) - { - for (dev = 0; dev < deviceCount; dev++) - { - cudaSetDevice(gpuids[dev]); - //Global index of FIRST projection on previous set on this GPU - proj_global=(i-1)*PROJ_PER_BLOCK+dev*nangles_device; - if (dev+1==deviceCount) { //is it the last device? - // projections assigned to this device is >=nangles_device-(deviceCount-1) and < nangles_device - if (i-1 < noOfKernelCallsLastDev) { - // The previous set(block) was not empty. - projection_this_block=min(PROJ_PER_BLOCK, nangles-proj_global); - } - else { - // The previous set was empty. - // This happens if deviceCount > PROJ_PER_BLOCK+1. - // e.g. PROJ_PER_BLOCK = 9, deviceCount = 11, nangles = 199. - // e.g. PROJ_PER_BLOCK = 1, deviceCount = 3, nangles = 7. - break; - } - } - else { - projection_this_block=PROJ_PER_BLOCK; - } - cudaMemcpyAsync(result[proj_global], dProjection[(int)(!(i%2))+dev*2], projection_this_block*geo.nDetecV*geo.nDetecU*sizeof(float), cudaMemcpyDeviceToHost,stream[dev*2+1]); - } - } - // Make sure Computation on kernels has finished before we launch the next batch. - for (dev = 0; dev < deviceCount; dev++) - { - cudaSetDevice(gpuids[dev]); - cudaStreamSynchronize(stream[dev*2]); - } - } // End noOfKernelCalls (i) loop. - - // We still have the last set of projections to get out of GPUs - for (dev = 0; dev < deviceCount; dev++) - { - cudaSetDevice(gpuids[dev]); - //Global index of FIRST projection on this set on this GPU - proj_global=(noOfKernelCalls-1)*PROJ_PER_BLOCK+dev*nangles_device; - if(proj_global>=nangles) - break; - // How many projections are left here? - projection_this_block=min(nangles_device-(noOfKernelCalls-1)*PROJ_PER_BLOCK, //the remaining angles that this GPU had to do (almost never PROJ_PER_BLOCK) - nangles-proj_global); //or whichever amount is left to finish all (this is for the last GPU) - - cudaDeviceSynchronize(); //Not really necessary, but just in case, we los nothing. - cudaCheckErrors("Error at copying the last set of projections out (or in the previous copy)"); - cudaMemcpyAsync(result[proj_global], dProjection[(int)(!(noOfKernelCalls%2))+dev*2], projection_this_block*geo.nDetecV*geo.nDetecU*sizeof(float), cudaMemcpyDeviceToHost,stream[dev*2+1]); - } - // Make sure everyone has done their bussiness before the next image split: - for (dev = 0; dev < deviceCount; dev++) - { - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - } - } // End image split loop. - - cudaCheckErrors("Main loop fail"); - /////////////////////////////////////////////////////////////////////// - /////////////////////////////////////////////////////////////////////// - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDestroyTextureObject(texImg[dev]); - cudaFreeArray(d_cuArrTex[dev]); - } - delete[] texImg; texImg = 0; - delete[] d_cuArrTex; d_cuArrTex = 0; - // Freeing Stage - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaFree(dProjection[dev*2]); - cudaFree(dProjection[dev*2+1]); - - } - free(dProjection); - - if(!fits_in_memory){ - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaFree(dProjection_accum[dev*2]); - cudaFree(dProjection_accum[dev*2+1]); - - } - free(dProjection_accum); - } - freeGeoArray(splits,geoArray); - cudaFreeHost(projParamsArrayHost); - cudaFreeHost(projFloatsArrayHost); - - - for (int i = 0; i < nStreams; ++i) - cudaStreamDestroy(stream[i]) ; -#ifndef NO_PINNED_MEMORY - if (isHostRegisterSupported & splits>1){ - cudaHostUnregister(img); - } -#endif - cudaCheckErrors("cudaFree fail"); - -// cudaDeviceReset(); - return 0; -} -void CreateTextureInterp(const GpuIds& gpuids,const float* imagedata,Geometry geo,cudaArray** d_cuArrTex, cudaTextureObject_t *texImage,bool allocate) -{ - const unsigned int num_devices = gpuids.GetLength(); - //size_t size_image=geo.nVoxelX*geo.nVoxelY*geo.nVoxelZ; - const cudaExtent extent = make_cudaExtent(geo.nVoxelX, geo.nVoxelY, geo.nVoxelZ); - if(allocate){ - - for (unsigned int dev = 0; dev < num_devices; dev++){ - cudaSetDevice(gpuids[dev]); - - //cudaArray Descriptor - - cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(); - //cuda Array - cudaMalloc3DArray(&d_cuArrTex[dev], &channelDesc, extent); - cudaCheckErrors("Texture memory allocation fail"); - } - - } - for (unsigned int dev = 0; dev < num_devices; dev++){ - cudaMemcpy3DParms copyParams = {0}; - cudaSetDevice(gpuids[dev]); - //Array creation - copyParams.srcPtr = make_cudaPitchedPtr((void *)imagedata, extent.width*sizeof(float), extent.width, extent.height); - copyParams.dstArray = d_cuArrTex[dev]; - copyParams.extent = extent; - copyParams.kind = cudaMemcpyHostToDevice; - cudaMemcpy3DAsync(©Params); - //cudaCheckErrors("Texture memory data copy fail"); - //Array creation End - } - for (unsigned int dev = 0; dev < num_devices; dev++){ - cudaSetDevice(gpuids[dev]); - cudaResourceDesc texRes; - memset(&texRes, 0, sizeof(cudaResourceDesc)); - texRes.resType = cudaResourceTypeArray; - texRes.res.array.array = d_cuArrTex[dev]; - cudaTextureDesc texDescr; - memset(&texDescr, 0, sizeof(cudaTextureDesc)); - texDescr.normalizedCoords = false; - if (geo.accuracy>1){ - texDescr.filterMode = cudaFilterModePoint; - geo.accuracy=1; - } - else{ - texDescr.filterMode = cudaFilterModeLinear; - } - texDescr.addressMode[0] = cudaAddressModeBorder; - texDescr.addressMode[1] = cudaAddressModeBorder; - texDescr.addressMode[2] = cudaAddressModeBorder; - texDescr.readMode = cudaReadModeElementType; - cudaCreateTextureObject(&texImage[dev], &texRes, &texDescr, NULL); - cudaCheckErrors("Texture object creation fail"); - } -} - -/* This code generates the geometries needed to split the image properly in - * cases where the entire image does not fit in the memory of the GPU - **/ -void splitImageInterp(unsigned int splits,Geometry geo,Geometry* geoArray, unsigned int nangles){ - - unsigned long splitsize=(geo.nVoxelZ+splits-1)/splits;// ceil if not divisible - for(unsigned int sp=0;spx=Pfinalu0.x-Pfinal.x; - deltaU->y=Pfinalu0.y-Pfinal.y; - deltaU->z=Pfinalu0.z-Pfinal.z; - - deltaV->x=Pfinalv0.x-Pfinal.x; - deltaV->y=Pfinalv0.y-Pfinal.y; - deltaV->z=Pfinalv0.z-Pfinal.z; - - *source=S; -} - -float maxdistanceCuboid(Geometry geo,unsigned int i){ - /////////// - // Compute initial "t" so we access safely as less as out of bounds as possible. - ////////// - - - float maxCubX,maxCubY,maxCubZ; - // Forgetting Z, compute mas distance: diagonal+offset - maxCubX=(geo.nVoxelX/2+ abs(geo.offOrigX[i])/geo.dVoxelX); - maxCubY=(geo.nVoxelY/2+ abs(geo.offOrigY[i])/geo.dVoxelY); - maxCubZ=(geo.nVoxelZ/2+ abs(geo.offOrigZ[i])/geo.dVoxelZ); - - float a,b; - a=geo.DSO[i]/geo.dVoxelX; - b=geo.DSO[i]/geo.dVoxelY; - -// As the return of this value is in "voxel space", the source may have an elliptical curve. -// The distance returned is the safe distance that can be skipped for a given angle alpha, before we need to start sampling. - - if (geo.theta==0.0f & geo.psi==0.0f) // Special case, it will make the code faster - return max(a*b/sqrt(a*a*sin(geo.alpha)*sin(geo.alpha)+b*b*cos(geo.alpha)*cos(geo.alpha))- - sqrt(maxCubX*maxCubX+maxCubY*maxCubY),0.0f); - //TODO: think of more special cases? - return max(geo.DSO[i]/max(max(geo.dVoxelX,geo.dVoxelY),geo.dVoxelZ)-sqrt(maxCubX*maxCubX+maxCubY*maxCubY+maxCubZ*maxCubZ),0.0f); - -} -void rollPitchYaw(Geometry geo,unsigned int i, Point3D* point){ - Point3D auxPoint; - auxPoint.x=point->x; - auxPoint.y=point->y; - auxPoint.z=point->z; - - point->x=cos(geo.dRoll[i])*cos(geo.dPitch[i])*auxPoint.x - +(cos(geo.dRoll[i])*sin(geo.dPitch[i])*sin(geo.dYaw[i]) - sin(geo.dRoll[i])*cos(geo.dYaw[i]))*auxPoint.y - +(cos(geo.dRoll[i])*sin(geo.dPitch[i])*cos(geo.dYaw[i]) + sin(geo.dRoll[i])*sin(geo.dYaw[i]))*auxPoint.z; - - point->y=sin(geo.dRoll[i])*cos(geo.dPitch[i])*auxPoint.x - +(sin(geo.dRoll[i])*sin(geo.dPitch[i])*sin(geo.dYaw[i]) + cos(geo.dRoll[i])*cos(geo.dYaw[i]))*auxPoint.y - +(sin(geo.dRoll[i])*sin(geo.dPitch[i])*cos(geo.dYaw[i]) - cos(geo.dRoll[i])*sin(geo.dYaw[i]))*auxPoint.z; - - point->z=-sin(geo.dPitch[i])*auxPoint.x - +cos(geo.dPitch[i])*sin(geo.dYaw[i])*auxPoint.y - +cos(geo.dPitch[i])*cos(geo.dYaw[i])*auxPoint.z; - -} -void eulerZYZ(Geometry geo, Point3D* point){ - Point3D auxPoint; - auxPoint.x=point->x; - auxPoint.y=point->y; - auxPoint.z=point->z; - - point->x=(+cos(geo.alpha)*cos(geo.theta)*cos(geo.psi)-sin(geo.alpha)*sin(geo.psi))*auxPoint.x+ - (-cos(geo.alpha)*cos(geo.theta)*sin(geo.psi)-sin(geo.alpha)*cos(geo.psi))*auxPoint.y+ - cos(geo.alpha)*sin(geo.theta)*auxPoint.z; - - point->y=(+sin(geo.alpha)*cos(geo.theta)*cos(geo.psi)+cos(geo.alpha)*sin(geo.psi))*auxPoint.x+ - (-sin(geo.alpha)*cos(geo.theta)*sin(geo.psi)+cos(geo.alpha)*cos(geo.psi))*auxPoint.y+ - sin(geo.alpha)*sin(geo.theta)*auxPoint.z; - - point->z=-sin(geo.theta)*cos(geo.psi)*auxPoint.x+ - sin(geo.theta)*sin(geo.psi)*auxPoint.y+ - cos(geo.theta)*auxPoint.z; - - -} -//______________________________________________________________________________ -// -// Function: freeGeoArray -// -// Description: Frees the memory from the geometry array for multiGPU. -//______________________________________________________________________________ -void freeGeoArray(unsigned int splits,Geometry* geoArray){ - for(unsigned int sp=0;sp +#include +#include +#include "ray_interpolated_projection.hpp" +#include "TIGRE_common.hpp" +#include + +#define hipCheckErrors(msg) \ +do { \ + hipError_t __err = hipGetLastError(); \ + if (__err != hipSuccess) { \ + mexPrintf("%s \n",msg);\ + hipDeviceReset();\ + mexErrMsgIdAndTxt("TIGRE:Ax:interpolated",hipGetErrorString(__err));\ + } \ +} while (0) + + + +#define MAXTREADS 1024 +#define PROJ_PER_BLOCK 9 +#define PIXEL_SIZE_BLOCK 9 + /*GEOMETRY DEFINITION + * + * Detector plane, behind + * |-----------------------------| + * | | + * | | + * | | + * | | + * | +--------+ | + * | / /| | + * A Z | / / |*D | + * | | +--------+ | | + * | | | | | | + * | | | *O | + | + * --->y | | | / | + * / | | |/ | + * V X | +--------+ | + * |-----------------------------| + * + * *S + * + * + * + * + * + **/ + void CreateTextureInterp(const GpuIds& gpuids,const float* imagedata,Geometry geo,hipArray** d_cuArrTex, hipTextureObject_t *texImage,bool allocate); +__constant__ Point3D projParamsArrayDev[4*PROJ_PER_BLOCK]; // Dev means it is on device +__constant__ float projFloatsArrayDev[2*PROJ_PER_BLOCK]; // Dev means it is on device + + +__global__ void vecAddInPlaceInterp(float *a, float *b, unsigned long n) +{ + int idx = blockIdx.x*blockDim.x+threadIdx.x; + // Make sure we do not go out of bounds + if (idx < n) + a[idx] = a[idx] + b[idx]; +} + + +template + __global__ void kernelPixelDetector( Geometry geo, + float* detector, + const int currProjSetNumber, + const int totalNoOfProjections, + hipTextureObject_t tex){ + + unsigned long long u = blockIdx.x * blockDim.x + threadIdx.x; + unsigned long long v = blockIdx.y * blockDim.y + threadIdx.y; + unsigned long long projNumber=threadIdx.z; + + if (u>= geo.nDetecU || v>= geo.nDetecV || projNumber>=PROJ_PER_BLOCK) + return; + +#if IS_FOR_MATLAB_TIGRE + size_t idx = (size_t)(u * (unsigned long long)geo.nDetecV + v)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; +#else + size_t idx = (size_t)(v * (unsigned long long)geo.nDetecU + u)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; +#endif + + unsigned long indAlpha = currProjSetNumber*PROJ_PER_BLOCK+projNumber; // This is the ABSOLUTE projection number in the projection array + + if(indAlpha>=totalNoOfProjections) + return; + + Point3D uvOrigin = projParamsArrayDev[4*projNumber]; // 6*projNumber because we have 6 Point3D values per projection + Point3D deltaU = projParamsArrayDev[4*projNumber+1]; + Point3D deltaV = projParamsArrayDev[4*projNumber+2]; + Point3D source = projParamsArrayDev[4*projNumber+3]; + + float DSO = projFloatsArrayDev[2*projNumber+0]; + float cropdist_init = projFloatsArrayDev[2*projNumber+1]; + + + + /////// Get coordinates XYZ of pixel UV + unsigned long pixelV = geo.nDetecV-v-1; + unsigned long pixelU = u; + + + float vectX,vectY,vectZ; + Point3D P; + P.x=(uvOrigin.x+pixelU*deltaU.x+pixelV*deltaV.x); + P.y=(uvOrigin.y+pixelU*deltaU.y+pixelV*deltaV.y); + P.z=(uvOrigin.z+pixelU*deltaU.z+pixelV*deltaV.z); + + // Length is the ray length in normalized space + float length=__fsqrt_rd((source.x-P.x)*(source.x-P.x)+(source.y-P.y)*(source.y-P.y)+(source.z-P.z)*(source.z-P.z)); + //now legth is an integer of Nsamples that are required on this line + length=ceilf(__fdividef(length,geo.accuracy));//Divide the directional vector by an integer + vectX=__fdividef(P.x -source.x,length); + vectY=__fdividef(P.y -source.y,length); + vectZ=__fdividef(P.z -source.z,length); + + +// //Integrate over the line + float tx,ty,tz; + float sum=0; + float i; + + + +// Because I have no idea how to efficiently cutoff the legth path in 3D, a very upper limit is computed (see maxdistanceCuboid) +// for the 3D case. However it would be bad to lose performance in the 3D case +// TODO: can ge really improve this? + if (sphericalrotation){ + if ((2*DSO/fminf(fminf(geo.dVoxelX,geo.dVoxelY),geo.dVoxelZ)+cropdist_init)/geo.accuracy < length) + length=ceilf((2*DSO/fminf(fminf(geo.dVoxelX,geo.dVoxelY),geo.dVoxelZ)+cropdist_init)/geo.accuracy); + } + else{ + if ((2*DSO/fminf(geo.dVoxelX,geo.dVoxelY)+cropdist_init)/geo.accuracy < length) + length=ceilf((2*DSO/fminf(geo.dVoxelX,geo.dVoxelY)+cropdist_init)/geo.accuracy); + } + + + //Length is not actually a length, but the amount of memreads with given accuracy ("samples per voxel") + for (i=floorf(cropdist_init/geo.accuracy); i<=length; i=i+1){ + tx=vectX*i+source.x; + ty=vectY*i+source.y; + tz=vectZ*i+source.z; + + sum += tex3D(tex, tx+0.5f, ty+0.5f, tz+0.5f); // this line is 94% of time. + } + + float deltalength=sqrtf((vectX*geo.dVoxelX)*(vectX*geo.dVoxelX)+ + (vectY*geo.dVoxelY)*(vectY*geo.dVoxelY)+ + (vectZ*geo.dVoxelZ)*(vectZ*geo.dVoxelZ) ); + + detector[idx]=sum*deltalength; +} + + + +// legnth(angles)=3 x nagnles, as we have roll, pitch, yaw. +int interpolation_projection(float * img, Geometry geo, float** result,float const * const angles,int nangles, const GpuIds& gpuids){ + + + // Prepare for MultiGPU + int deviceCount = gpuids.GetLength(); + hipCheckErrors("Device query fail"); + if (deviceCount == 0) { + mexErrMsgIdAndTxt("Ax:Interpolated_projection:GPUselect","There are no available device(s) that support CUDA\n"); + } + // + // CODE assumes + // 1.-All available devices are usable by this code + // 2.-All available devices are equal, they are the same machine (warning thrown) + // Check the available devices, and if they are the same + if (!gpuids.AreEqualDevices()) { + mexWarnMsgIdAndTxt("Ax:Interpolated_projection:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); + } + int dev; + + // Check free memory + size_t mem_GPU_global; + checkFreeMemory(gpuids,&mem_GPU_global); + + // printf("geo.nDetec (U, V) = %d, %d\n", geo.nDetecU, geo.nDetecV); + + size_t mem_image=(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY*(unsigned long long)geo.nVoxelZ*sizeof(float); + size_t mem_proj =(unsigned long long)geo.nDetecU*(unsigned long long)geo.nDetecV * sizeof(float); + + // Does everything fit in the GPUs? + const bool fits_in_memory = mem_image+2*PROJ_PER_BLOCK*mem_proj= 9020 + hipDeviceGetAttribute(&isHostRegisterSupported,hipDevAttrHostRegisterSupported,gpuids[0]); +#endif + // empirical testing shows that when the image split is smaller than 1 (also implies the image is not very big), the time to + // pin the memory is greater than the lost time in Synchronously launching the memcpys. This is only worth it when the image is too big. + +#ifndef NO_PINNED_MEMORY + if (isHostRegisterSupported & splits>1){ + hipHostRegister(img, (size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geo.nVoxelZ*(size_t)sizeof(float),hipHostRegisterPortable); + } + hipCheckErrors("Error pinning memory"); +#endif + Point3D source, deltaU, deltaV, uvOrigin; + + Point3D* projParamsArrayHost = 0; + hipMallocHost((void**)&projParamsArrayHost,4*PROJ_PER_BLOCK*sizeof(Point3D)); + float* projFloatsArrayHost = 0; + hipMallocHost((void**)&projFloatsArrayHost,2*PROJ_PER_BLOCK*sizeof(float)); + hipCheckErrors("Error allocating auxiliary constant memory"); + + // Create Streams for overlapping memcopy and compute + int nStream_device=2; + int nStreams=deviceCount*nStream_device; + hipStream_t* stream=(hipStream_t*)malloc(nStreams*sizeof(hipStream_t)); + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + for (int i = 0; i < nStream_device; ++i){ + hipStreamCreate(&stream[i+dev*nStream_device]); + + } + } + hipCheckErrors("Stream creation fail"); + int nangles_device=(nangles+deviceCount-1)/deviceCount; + int nangles_last_device=(nangles-(deviceCount-1)*nangles_device); + unsigned int noOfKernelCalls = (nangles_device+PROJ_PER_BLOCK-1)/PROJ_PER_BLOCK; // We'll take care of bounds checking inside the loop if nalpha is not divisible by PROJ_PER_BLOCK + unsigned int noOfKernelCallsLastDev = (nangles_last_device+PROJ_PER_BLOCK-1)/PROJ_PER_BLOCK; // we will use this in the memory management. + int projection_this_block; + + + + hipTextureObject_t *texImg = new hipTextureObject_t[deviceCount]; + hipArray **d_cuArrTex = new hipArray*[deviceCount]; + for (unsigned int sp=0;sp=nangles) + break; + if ((i*PROJ_PER_BLOCK+j)>=nangles_device) + break; + geoArray[sp].alpha=angles[proj_global*3]; + geoArray[sp].theta=angles[proj_global*3+1]; + geoArray[sp].psi =angles[proj_global*3+2]; + + is_spherical+=abs(geoArray[sp].theta)+abs(geoArray[sp].psi); + + //precomute distances for faster execution + maxdist=maxdistanceCuboid(geoArray[sp],proj_global); + //Precompute per angle constant stuff for speed + computeDeltas(geoArray[sp], proj_global, &uvOrigin, &deltaU, &deltaV, &source); + //Ray tracing! + projParamsArrayHost[4*j]=uvOrigin; // 6*j because we have 6 Point3D values per projection + projParamsArrayHost[4*j+1]=deltaU; + projParamsArrayHost[4*j+2]=deltaV; + projParamsArrayHost[4*j+3]=source; + + projFloatsArrayHost[2*j]=geo.DSO[proj_global]; + projFloatsArrayHost[2*j+1]=floor(maxdist); + } + + hipMemcpyToSymbolAsync(projParamsArrayDev, projParamsArrayHost, sizeof(Point3D)*4*PROJ_PER_BLOCK,0,hipMemcpyHostToDevice,stream[dev*nStream_device]); + hipMemcpyToSymbolAsync(projFloatsArrayDev, projFloatsArrayHost, sizeof(float)*2*PROJ_PER_BLOCK,0,hipMemcpyHostToDevice,stream[dev*nStream_device]); + hipStreamSynchronize(stream[dev*nStream_device]); + + + //TODO: we could do this around X and Y axis too, but we would need to compute the new axis of rotation (not possible to know from jsut the angles) + if (!is_spherical){ + kernelPixelDetector<<>>(geoArray[sp],dProjection[(i%2)+dev*2],i,nangles_device,texImg[dev]); + } + else{ + kernelPixelDetector <<>>(geoArray[sp],dProjection[(i%2)+dev*2],i,nangles_device,texImg[dev]); + } + } + + + // Now that the computation is happening, we need to either prepare the memory for + // combining of the projections (splits>1) and start removing previous results. + + + // If our image does not fit in memory then we need to make sure we accumulate previous results too. + // This is done in 2 steps: + // 1)copy previous results back into GPU + // 2)accumulate with current results + // The code to take them out is the same as when there are no splits needed + if( !fits_in_memory&&sp>0) + { + // 1) grab previous results and put them in the auxiliary variable dProjection_accum + for (dev = 0; dev < deviceCount; dev++) + { + hipSetDevice(gpuids[dev]); + //Global index of FIRST projection on this set on this GPU + proj_global=i*PROJ_PER_BLOCK+dev*nangles_device; + if(proj_global>=nangles) + break; + + // Unless its the last projection set, we have PROJ_PER_BLOCK angles. Otherwise... + if(i+1==noOfKernelCalls) //is it the last block? + projection_this_block=min(nangles_device-(noOfKernelCalls-1)*PROJ_PER_BLOCK, //the remaining angles that this GPU had to do (almost never PROJ_PER_BLOCK) + nangles-proj_global); //or whichever amount is left to finish all (this is for the last GPU) + else + projection_this_block=PROJ_PER_BLOCK; + hipMemcpyAsync(dProjection_accum[(i%2)+dev*2], result[proj_global], projection_this_block*geo.nDetecV*geo.nDetecU*sizeof(float), hipMemcpyHostToDevice,stream[dev*2+1]); + } + // 2) take the results from current compute call and add it to the code in execution. + for (dev = 0; dev < deviceCount; dev++) + { + hipSetDevice(gpuids[dev]); + //Global index of FIRST projection on this set on this GPU + proj_global=i*PROJ_PER_BLOCK+dev*nangles_device; + if(proj_global>=nangles) + break; + + // Unless its the last projection set, we have PROJ_PER_BLOCK angles. Otherwise... + if(i+1==noOfKernelCalls) //is it the last block? + projection_this_block=min(nangles_device-(noOfKernelCalls-1)*PROJ_PER_BLOCK, //the remaining angles that this GPU had to do (almost never PROJ_PER_BLOCK) + nangles-proj_global); //or whichever amount is left to finish all (this is for the last GPU) + else + projection_this_block=PROJ_PER_BLOCK; + hipStreamSynchronize(stream[dev*2+1]); // wait until copy is finished + vecAddInPlaceInterp<<<(geo.nDetecU*geo.nDetecV*projection_this_block+MAXTREADS-1)/MAXTREADS,MAXTREADS,0,stream[dev*2]>>>(dProjection[(i%2)+dev*2],dProjection_accum[(i%2)+dev*2],(unsigned long)geo.nDetecU*geo.nDetecV*projection_this_block); + } + } // end accumulation case, where the image needs to be split + + // Now, lets get out the projections from the previous execution of the kernels. + if (i>0) + { + for (dev = 0; dev < deviceCount; dev++) + { + hipSetDevice(gpuids[dev]); + //Global index of FIRST projection on previous set on this GPU + proj_global=(i-1)*PROJ_PER_BLOCK+dev*nangles_device; + if (dev+1==deviceCount) { //is it the last device? + // projections assigned to this device is >=nangles_device-(deviceCount-1) and < nangles_device + if (i-1 < noOfKernelCallsLastDev) { + // The previous set(block) was not empty. + projection_this_block=min(PROJ_PER_BLOCK, nangles-proj_global); + } + else { + // The previous set was empty. + // This happens if deviceCount > PROJ_PER_BLOCK+1. + // e.g. PROJ_PER_BLOCK = 9, deviceCount = 11, nangles = 199. + // e.g. PROJ_PER_BLOCK = 1, deviceCount = 3, nangles = 7. + break; + } + } + else { + projection_this_block=PROJ_PER_BLOCK; + } + hipMemcpyAsync(result[proj_global], dProjection[(int)(!(i%2))+dev*2], projection_this_block*geo.nDetecV*geo.nDetecU*sizeof(float), hipMemcpyDeviceToHost,stream[dev*2+1]); + } + } + // Make sure Computation on kernels has finished before we launch the next batch. + for (dev = 0; dev < deviceCount; dev++) + { + hipSetDevice(gpuids[dev]); + hipStreamSynchronize(stream[dev*2]); + } + } // End noOfKernelCalls (i) loop. + + // We still have the last set of projections to get out of GPUs + for (dev = 0; dev < deviceCount; dev++) + { + hipSetDevice(gpuids[dev]); + //Global index of FIRST projection on this set on this GPU + proj_global=(noOfKernelCalls-1)*PROJ_PER_BLOCK+dev*nangles_device; + if(proj_global>=nangles) + break; + // How many projections are left here? + projection_this_block=min(nangles_device-(noOfKernelCalls-1)*PROJ_PER_BLOCK, //the remaining angles that this GPU had to do (almost never PROJ_PER_BLOCK) + nangles-proj_global); //or whichever amount is left to finish all (this is for the last GPU) + + hipDeviceSynchronize(); //Not really necessary, but just in case, we los nothing. + hipCheckErrors("Error at copying the last set of projections out (or in the previous copy)"); + hipMemcpyAsync(result[proj_global], dProjection[(int)(!(noOfKernelCalls%2))+dev*2], projection_this_block*geo.nDetecV*geo.nDetecU*sizeof(float), hipMemcpyDeviceToHost,stream[dev*2+1]); + } + // Make sure everyone has done their bussiness before the next image split: + for (dev = 0; dev < deviceCount; dev++) + { + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + } + } // End image split loop. + + hipCheckErrors("Main loop fail"); + /////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////// + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDestroyTextureObject(texImg[dev]); + hipFreeArray(d_cuArrTex[dev]); + } + delete[] texImg; texImg = 0; + delete[] d_cuArrTex; d_cuArrTex = 0; + // Freeing Stage + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipFree(dProjection[dev*2]); + hipFree(dProjection[dev*2+1]); + + } + free(dProjection); + + if(!fits_in_memory){ + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipFree(dProjection_accum[dev*2]); + hipFree(dProjection_accum[dev*2+1]); + + } + free(dProjection_accum); + } + freeGeoArray(splits,geoArray); + hipFreeHost(projParamsArrayHost); + hipFreeHost(projFloatsArrayHost); + + + for (int i = 0; i < nStreams; ++i) + hipStreamDestroy(stream[i]) ; +#ifndef NO_PINNED_MEMORY + if (isHostRegisterSupported & splits>1){ + hipHostUnregister(img); + } +#endif + hipCheckErrors("hipFree fail"); + +// hipDeviceReset(); + return 0; +} +void CreateTextureInterp(const GpuIds& gpuids,const float* imagedata,Geometry geo,hipArray** d_cuArrTex, hipTextureObject_t *texImage,bool allocate) +{ + const unsigned int num_devices = gpuids.GetLength(); + //size_t size_image=geo.nVoxelX*geo.nVoxelY*geo.nVoxelZ; + const hipExtent extent = make_cudaExtent(geo.nVoxelX, geo.nVoxelY, geo.nVoxelZ); + if(allocate){ + + for (unsigned int dev = 0; dev < num_devices; dev++){ + hipSetDevice(gpuids[dev]); + + //hipArray Descriptor + + hipChannelFormatDesc channelDesc = hipCreateChannelDesc(); + //cuda Array + hipMalloc3DArray(&d_cuArrTex[dev], &channelDesc, extent); + hipCheckErrors("Texture memory allocation fail"); + } + + } + for (unsigned int dev = 0; dev < num_devices; dev++){ + hipMemcpy3DParms copyParams = {0}; + hipSetDevice(gpuids[dev]); + //Array creation + copyParams.srcPtr = make_cudaPitchedPtr((void *)imagedata, extent.width*sizeof(float), extent.width, extent.height); + copyParams.dstArray = d_cuArrTex[dev]; + copyParams.extent = extent; + copyParams.kind = hipMemcpyHostToDevice; + hipMemcpy3DAsync(©Params); + //hipCheckErrors("Texture memory data copy fail"); + //Array creation End + } + for (unsigned int dev = 0; dev < num_devices; dev++){ + hipSetDevice(gpuids[dev]); + hipResourceDesc texRes; + memset(&texRes, 0, sizeof(hipResourceDesc)); + texRes.resType = hipResourceTypeArray; + texRes.res.array.array = d_cuArrTex[dev]; + hipTextureDesc texDescr; + memset(&texDescr, 0, sizeof(hipTextureDesc)); + texDescr.normalizedCoords = false; + if (geo.accuracy>1){ + texDescr.filterMode = hipFilterModePoint; + geo.accuracy=1; + } + else{ + texDescr.filterMode = hipFilterModeLinear; + } + texDescr.addressMode[0] = hipAddressModeBorder; + texDescr.addressMode[1] = hipAddressModeBorder; + texDescr.addressMode[2] = hipAddressModeBorder; + texDescr.readMode = hipReadModeElementType; + hipCreateTextureObject(&texImage[dev], &texRes, &texDescr, NULL); + hipCheckErrors("Texture object creation fail"); + } +} + +/* This code generates the geometries needed to split the image properly in + * cases where the entire image does not fit in the memory of the GPU + **/ +void splitImageInterp(unsigned int splits,Geometry geo,Geometry* geoArray, unsigned int nangles){ + + unsigned long splitsize=(geo.nVoxelZ+splits-1)/splits;// ceil if not divisible + for(unsigned int sp=0;spx=Pfinalu0.x-Pfinal.x; + deltaU->y=Pfinalu0.y-Pfinal.y; + deltaU->z=Pfinalu0.z-Pfinal.z; + + deltaV->x=Pfinalv0.x-Pfinal.x; + deltaV->y=Pfinalv0.y-Pfinal.y; + deltaV->z=Pfinalv0.z-Pfinal.z; + + *source=S; +} + +float maxdistanceCuboid(Geometry geo,unsigned int i){ + /////////// + // Compute initial "t" so we access safely as less as out of bounds as possible. + ////////// + + + float maxCubX,maxCubY,maxCubZ; + // Forgetting Z, compute mas distance: diagonal+offset + maxCubX=(geo.nVoxelX/2+ abs(geo.offOrigX[i])/geo.dVoxelX); + maxCubY=(geo.nVoxelY/2+ abs(geo.offOrigY[i])/geo.dVoxelY); + maxCubZ=(geo.nVoxelZ/2+ abs(geo.offOrigZ[i])/geo.dVoxelZ); + + float a,b; + a=geo.DSO[i]/geo.dVoxelX; + b=geo.DSO[i]/geo.dVoxelY; + +// As the return of this value is in "voxel space", the source may have an elliptical curve. +// The distance returned is the safe distance that can be skipped for a given angle alpha, before we need to start sampling. + + if (geo.theta==0.0f & geo.psi==0.0f) // Special case, it will make the code faster + return max(a*b/sqrt(a*a*sin(geo.alpha)*sin(geo.alpha)+b*b*cos(geo.alpha)*cos(geo.alpha))- + sqrt(maxCubX*maxCubX+maxCubY*maxCubY),0.0f); + //TODO: think of more special cases? + return max(geo.DSO[i]/max(max(geo.dVoxelX,geo.dVoxelY),geo.dVoxelZ)-sqrt(maxCubX*maxCubX+maxCubY*maxCubY+maxCubZ*maxCubZ),0.0f); + +} +void rollPitchYaw(Geometry geo,unsigned int i, Point3D* point){ + Point3D auxPoint; + auxPoint.x=point->x; + auxPoint.y=point->y; + auxPoint.z=point->z; + + point->x=cos(geo.dRoll[i])*cos(geo.dPitch[i])*auxPoint.x + +(cos(geo.dRoll[i])*sin(geo.dPitch[i])*sin(geo.dYaw[i]) - sin(geo.dRoll[i])*cos(geo.dYaw[i]))*auxPoint.y + +(cos(geo.dRoll[i])*sin(geo.dPitch[i])*cos(geo.dYaw[i]) + sin(geo.dRoll[i])*sin(geo.dYaw[i]))*auxPoint.z; + + point->y=sin(geo.dRoll[i])*cos(geo.dPitch[i])*auxPoint.x + +(sin(geo.dRoll[i])*sin(geo.dPitch[i])*sin(geo.dYaw[i]) + cos(geo.dRoll[i])*cos(geo.dYaw[i]))*auxPoint.y + +(sin(geo.dRoll[i])*sin(geo.dPitch[i])*cos(geo.dYaw[i]) - cos(geo.dRoll[i])*sin(geo.dYaw[i]))*auxPoint.z; + + point->z=-sin(geo.dPitch[i])*auxPoint.x + +cos(geo.dPitch[i])*sin(geo.dYaw[i])*auxPoint.y + +cos(geo.dPitch[i])*cos(geo.dYaw[i])*auxPoint.z; + +} +void eulerZYZ(Geometry geo, Point3D* point){ + Point3D auxPoint; + auxPoint.x=point->x; + auxPoint.y=point->y; + auxPoint.z=point->z; + + point->x=(+cos(geo.alpha)*cos(geo.theta)*cos(geo.psi)-sin(geo.alpha)*sin(geo.psi))*auxPoint.x+ + (-cos(geo.alpha)*cos(geo.theta)*sin(geo.psi)-sin(geo.alpha)*cos(geo.psi))*auxPoint.y+ + cos(geo.alpha)*sin(geo.theta)*auxPoint.z; + + point->y=(+sin(geo.alpha)*cos(geo.theta)*cos(geo.psi)+cos(geo.alpha)*sin(geo.psi))*auxPoint.x+ + (-sin(geo.alpha)*cos(geo.theta)*sin(geo.psi)+cos(geo.alpha)*cos(geo.psi))*auxPoint.y+ + sin(geo.alpha)*sin(geo.theta)*auxPoint.z; + + point->z=-sin(geo.theta)*cos(geo.psi)*auxPoint.x+ + sin(geo.theta)*sin(geo.psi)*auxPoint.y+ + cos(geo.theta)*auxPoint.z; + + +} +//______________________________________________________________________________ +// +// Function: freeGeoArray +// +// Description: Frees the memory from the geometry array for multiGPU. +//______________________________________________________________________________ +void freeGeoArray(unsigned int splits,Geometry* geoArray){ + for(unsigned int sp=0;sp -#include -#include -#include "ray_interpolated_projection_parallel.hpp" -#include "TIGRE_common.hpp" -#include - -#define cudaCheckErrors(msg) \ -do { \ - cudaError_t __err = cudaGetLastError(); \ - if (__err != cudaSuccess) { \ - mexPrintf("%s \n",msg);\ - mexErrMsgIdAndTxt("TIGRE:Ax:interpolated_parallel",cudaGetErrorString(__err));\ - } \ -} while (0) - - - -#define MAXTREADS 1024 -#define PROJ_PER_BLOCK 8 -#define PIXEL_SIZE_BLOCK 8 -/*GEOMETRY DEFINITION - * - * Detector plane, behind - * |-----------------------------| - * | | - * | | - * | | - * | | - * | +--------+ | - * | / /| | - * A Z | / / |*D | - * | | +--------+ | | - * | | | | | | - * | | | *O | + | - * --->y | | | / | - * / | | |/ | - * V X | +--------+ | - * |-----------------------------| - * - * *S - * - * - * - * - * - **/ -void CreateTextureParallelInterp(float* image,Geometry geo,cudaArray** d_cuArrTex, cudaTextureObject_t *texImage,cudaStream_t* stream); -__constant__ Point3D projParamsArrayDev[4*PROJ_PER_BLOCK]; // Dev means it is on device -__constant__ float projFloatsArrayDev[2*PROJ_PER_BLOCK]; // Dev means it is on device - - - -__global__ void kernelPixelDetector_parallel_interpolated( Geometry geo, - float* detector, - const int currProjSetNumber, const int totalNoOfProjections, cudaTextureObject_t tex) -{ -// Point3D source , -// Point3D deltaU, -// Point3D deltaV, -// Point3D uvOrigin, -// float DSO, -// float maxdist){ - - unsigned long long u = blockIdx.x * blockDim.x + threadIdx.x; - unsigned long long v = blockIdx.y * blockDim.y + threadIdx.y; - unsigned long long projNumber=threadIdx.z; - - if (u>= geo.nDetecU || v>= geo.nDetecV || projNumber>=PROJ_PER_BLOCK) - return; - - int indAlpha = currProjSetNumber*PROJ_PER_BLOCK+projNumber; // This is the ABSOLUTE projection number in the projection array - - -#if IS_FOR_MATLAB_TIGRE - size_t idx = (size_t)(u * (unsigned long long)geo.nDetecV + v)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; -#else - size_t idx = (size_t)(v * (unsigned long long)geo.nDetecU + u)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; -#endif - - if(indAlpha>=totalNoOfProjections) - return; - - Point3D uvOrigin = projParamsArrayDev[4*projNumber]; // 6*projNumber because we have 6 Point3D values per projection - Point3D deltaU = projParamsArrayDev[4*projNumber+1]; - Point3D deltaV = projParamsArrayDev[4*projNumber+2]; - Point3D source = projParamsArrayDev[4*projNumber+3]; - - float DSO = projFloatsArrayDev[2*projNumber+0]; - float maxdist = projFloatsArrayDev[2*projNumber+1]; - - - /////// Get coordinates XYZ of pixel UV - unsigned long pixelV = geo.nDetecV-v-1; - unsigned long pixelU = u; - - - float vectX,vectY,vectZ; - Point3D P; - P.x=(uvOrigin.x+pixelU*deltaU.x+pixelV*deltaV.x); - P.y=(uvOrigin.y+pixelU*deltaU.y+pixelV*deltaV.y); - P.z=(uvOrigin.z+pixelU*deltaU.z+pixelV*deltaV.z); - Point3D S; - S.x=(source.x+pixelU*deltaU.x+pixelV*deltaV.x); - S.y=(source.y+pixelU*deltaU.y+pixelV*deltaV.y); - S.z=(source.z+pixelU*deltaU.z+pixelV*deltaV.z); - - // Length is the ray length in normalized space - double length=sqrtf((S.x-P.x)*(S.x-P.x)+(S.y-P.y)*(S.y-P.y)+(S.z-P.z)*(S.z-P.z)); - //now legth is an integer of Nsamples that are required on this line - length=ceilf(length/geo.accuracy);//Divide the directional vector by an integer - vectX=(P.x -S.x)/(length); - vectY=(P.y -S.y)/(length); - vectZ=(P.z -S.z)/(length); - - -// //Integrate over the line - float tx,ty,tz; - float sum=0; - float i; - - - // limit the amount of mem access after the cube, but before the detector. - if ((2*DSO/geo.dVoxelX+maxdist)/geo.accuracy < length) - length=ceilf((2*DSO/geo.dVoxelX+maxdist)/geo.accuracy); - //Length is not actually a length, but the amount of memreads with given accuracy ("samples per voxel") - - for (i=floorf(maxdist/geo.accuracy); i<=length; i=i+1){ - tx=vectX*i+S.x; - ty=vectY*i+S.y; - tz=vectZ*i+S.z; - - sum += tex3D(tex, tx+0.5f, ty+0.5f, tz+0.5f); // this line is 94% of time. - - } - float deltalength=sqrtf((vectX*geo.dVoxelX)*(vectX*geo.dVoxelX)+ - (vectY*geo.dVoxelY)*(vectY*geo.dVoxelY)+ - (vectZ*geo.dVoxelZ)*(vectZ*geo.dVoxelZ) ); - detector[idx]=sum*deltalength; -} - - - -int interpolation_projection_parallel(float * img, Geometry geo, float** result,float const * const angles,int nangles, const GpuIds& gpuids){ - - - - size_t num_bytes = geo.nDetecU*geo.nDetecV *PROJ_PER_BLOCK* sizeof(float); - float** dProjection=(float **)malloc(2*sizeof(float *)); - for (int i = 0; i < 2; ++i){ - cudaMalloc((void**)&dProjection[i], num_bytes); - cudaCheckErrors("cudaMalloc projections fail"); - } - // allocate streams for memory and compute - int nStreams=2; - cudaStream_t* stream=(cudaStream_t*)malloc(nStreams*sizeof(cudaStream_t));; - - for (int i = 0; i < 2; ++i){ - cudaStreamCreate(&stream[i]); - } - - - // Texture object variables - cudaTextureObject_t *texImg = 0; - cudaArray **d_cuArrTex = 0; - texImg =(cudaTextureObject_t*)malloc(1*sizeof(cudaTextureObject_t)); - d_cuArrTex =(cudaArray**)malloc(1*sizeof(cudaArray*)); - - CreateTextureParallelInterp(img,geo,&d_cuArrTex[0], &texImg[0],stream); - cudaCheckErrors("Texture allocation fail"); - //Done! Image put into texture memory. - - - - Point3D source, deltaU, deltaV, uvOrigin; - - Point3D* projParamsArrayHost; - cudaMallocHost((void**)&projParamsArrayHost,4*PROJ_PER_BLOCK*sizeof(Point3D)); - float* projFloatsArrayHost; - cudaMallocHost((void**)&projFloatsArrayHost,2*PROJ_PER_BLOCK*sizeof(float)); - - // 16x16 gave the best performance empirically - // Funnily that makes it compatible with most GPUs..... - int divU,divV,divangle; - divU=PIXEL_SIZE_BLOCK; - divV=PIXEL_SIZE_BLOCK; - - dim3 numBlocks((geo.nDetecU+divU-1)/divU,(geo.nDetecV+divV-1)/divV,1); - dim3 threadsPerBlock(divU,divV,PROJ_PER_BLOCK); - unsigned int proj_global; - unsigned int noOfKernelCalls = (nangles+PROJ_PER_BLOCK-1)/PROJ_PER_BLOCK; // We'll take care of bounds checking inside the loop if nalpha is not divisible by PROJ_PER_BLOCK - unsigned int i; - - float maxdist; - for ( i=0; i=nangles) - break; - - geo.alpha=angles[proj_global*3]; - geo.theta=angles[proj_global*3+1]; - geo.psi =angles[proj_global*3+2]; - //precomute distances for faster execution - maxdist=maxdistanceCuboid(geo,proj_global); - //Precompute per angle constant stuff for speed - computeDeltas_parallel(geo,geo.alpha,proj_global, &uvOrigin, &deltaU, &deltaV, &source); - //Ray tracing! - projParamsArrayHost[4*j]=uvOrigin; // 6*j because we have 6 Point3D values per projection - projParamsArrayHost[4*j+1]=deltaU; - projParamsArrayHost[4*j+2]=deltaV; - projParamsArrayHost[4*j+3]=source; - - projFloatsArrayHost[2*j]=geo.DSO[proj_global]; - projFloatsArrayHost[2*j+1]=floor(maxdist); - - } - cudaMemcpyToSymbolAsync(projParamsArrayDev, projParamsArrayHost, sizeof(Point3D)*4*PROJ_PER_BLOCK,0,cudaMemcpyHostToDevice,stream[0]); - cudaMemcpyToSymbolAsync(projFloatsArrayDev, projFloatsArrayHost, sizeof(float)*2*PROJ_PER_BLOCK,0,cudaMemcpyHostToDevice,stream[0]); - cudaStreamSynchronize(stream[0]); - - kernelPixelDetector_parallel_interpolated<<>>(geo,dProjection[(int)i%2==0],i,nangles,texImg[0]); - // copy result to host - if (i>0) - cudaMemcpyAsync(result[i*PROJ_PER_BLOCK-PROJ_PER_BLOCK],dProjection[(int)i%2!=0], num_bytes, cudaMemcpyDeviceToHost,stream[1]); - } - cudaDeviceSynchronize(); - - int lastangles=nangles-(i-1)*PROJ_PER_BLOCK; - cudaMemcpyAsync(result[(i-1)*PROJ_PER_BLOCK],dProjection[(int)(i-1)%2==0], lastangles*geo.nDetecV*geo.nDetecU*sizeof(float), cudaMemcpyDeviceToHost,stream[1]); - - - cudaDestroyTextureObject(texImg[0]); - cudaFreeArray(d_cuArrTex[0]); - free(texImg); texImg = 0; - free(d_cuArrTex); d_cuArrTex = 0; - cudaCheckErrors("Unbind fail"); - cudaFree(dProjection[0]); - cudaFree(dProjection[1]); - free(dProjection); - cudaFreeHost(projParamsArrayHost); - cudaFreeHost(projFloatsArrayHost); - - cudaCheckErrors("cudaFree d_imagedata fail"); - - - for (int i = 0; i < 2; ++i){ - cudaStreamDestroy(stream[i]); - } -// cudaDeviceReset(); - - return 0; -} - - - - -/* This code precomputes The location of the source and the Delta U and delta V (in the warped space) - * to compute the locations of the x-rays. While it seems verbose and overly-optimized, - * it does saves about 30% of each of the kernel calls. Thats something! - **/ -void computeDeltas_parallel(Geometry geo, float alpha,unsigned int i, Point3D* uvorigin, Point3D* deltaU, Point3D* deltaV, Point3D* source){ - Point3D S; - S.x=geo.DSO[i]; - S.y=geo.dDetecU*(0-((float)geo.nDetecU/2)+0.5); - S.z=geo.dDetecV*(((float)geo.nDetecV/2)-0.5-0); - - //End point - Point3D P,Pu0,Pv0; - - P.x =-(geo.DSD[i]-geo.DSO[i]); P.y = geo.dDetecU*(0-((float)geo.nDetecU/2)+0.5); P.z = geo.dDetecV*(((float)geo.nDetecV/2)-0.5-0); - Pu0.x=-(geo.DSD[i]-geo.DSO[i]); Pu0.y= geo.dDetecU*(1-((float)geo.nDetecU/2)+0.5); Pu0.z= geo.dDetecV*(((float)geo.nDetecV/2)-0.5-0); - Pv0.x=-(geo.DSD[i]-geo.DSO[i]); Pv0.y= geo.dDetecU*(0-((float)geo.nDetecU/2)+0.5); Pv0.z= geo.dDetecV*(((float)geo.nDetecV/2)-0.5-1); - // Geometric trasnformations: - P.x=0;Pu0.x=0;Pv0.x=0; - - // Roll pitch yaw - rollPitchYaw(geo,i,&P); - rollPitchYaw(geo,i,&Pu0); - rollPitchYaw(geo,i,&Pv0); - //Now lets translate the points where they should be: - P.x=P.x-(geo.DSD[i]-geo.DSO[i]); - Pu0.x=Pu0.x-(geo.DSD[i]-geo.DSO[i]); - Pv0.x=Pv0.x-(geo.DSD[i]-geo.DSO[i]); - - S.x=0; - // Roll pitch yaw - rollPitchYaw(geo,i,&S); - //Now lets translate the points where they should be: - S.x=S.x+geo.DSO[i]; - - - //1: Offset detector - - //P.x - P.y =P.y +geo.offDetecU[i]; P.z =P.z +geo.offDetecV[i]; - Pu0.y=Pu0.y+geo.offDetecU[i]; Pu0.z=Pu0.z+geo.offDetecV[i]; - Pv0.y=Pv0.y+geo.offDetecU[i]; Pv0.z=Pv0.z+geo.offDetecV[i]; - //S doesnt need to chagne - - - //3: Rotate (around z)! - Point3D Pfinal, Pfinalu0, Pfinalv0; - Pfinal.x =P.x; - Pfinal.y =P.y +geo.offDetecU[i]; Pfinal.z =P.z +geo.offDetecV[i]; - Pfinalu0.x=Pu0.x; - Pfinalu0.y=Pu0.y +geo.offDetecU[i]; Pfinalu0.z =Pu0.z +geo.offDetecV[i]; - Pfinalv0.x=Pv0.x; - Pfinalv0.y=Pv0.y +geo.offDetecU[i]; Pfinalv0.z =Pv0.z +geo.offDetecV[i]; - - eulerZYZ(geo,&Pfinal); - eulerZYZ(geo,&Pfinalu0); - eulerZYZ(geo,&Pfinalv0); - eulerZYZ(geo,&S); - - - - //2: Offset image (instead of offseting image, -offset everything else) - - Pfinal.x =Pfinal.x-geo.offOrigX[i]; Pfinal.y =Pfinal.y-geo.offOrigY[i]; Pfinal.z =Pfinal.z-geo.offOrigZ[i]; - Pfinalu0.x=Pfinalu0.x-geo.offOrigX[i]; Pfinalu0.y=Pfinalu0.y-geo.offOrigY[i]; Pfinalu0.z=Pfinalu0.z-geo.offOrigZ[i]; - Pfinalv0.x=Pfinalv0.x-geo.offOrigX[i]; Pfinalv0.y=Pfinalv0.y-geo.offOrigY[i]; Pfinalv0.z=Pfinalv0.z-geo.offOrigZ[i]; - S.x=S.x-geo.offOrigX[i]; S.y=S.y-geo.offOrigY[i]; S.z=S.z-geo.offOrigZ[i]; - - // As we want the (0,0,0) to be in a corner of the image, we need to translate everything (after rotation); - Pfinal.x =Pfinal.x+geo.sVoxelX/2-geo.dVoxelX/2; Pfinal.y =Pfinal.y+geo.sVoxelY/2-geo.dVoxelY/2; Pfinal.z =Pfinal.z +geo.sVoxelZ/2-geo.dVoxelZ/2; - Pfinalu0.x=Pfinalu0.x+geo.sVoxelX/2-geo.dVoxelX/2; Pfinalu0.y=Pfinalu0.y+geo.sVoxelY/2-geo.dVoxelY/2; Pfinalu0.z=Pfinalu0.z+geo.sVoxelZ/2-geo.dVoxelZ/2; - Pfinalv0.x=Pfinalv0.x+geo.sVoxelX/2-geo.dVoxelX/2; Pfinalv0.y=Pfinalv0.y+geo.sVoxelY/2-geo.dVoxelY/2; Pfinalv0.z=Pfinalv0.z+geo.sVoxelZ/2-geo.dVoxelZ/2; - S.x =S.x+geo.sVoxelX/2-geo.dVoxelX/2; S.y =S.y+geo.sVoxelY/2-geo.dVoxelY/2; S.z =S.z +geo.sVoxelZ/2-geo.dVoxelZ/2; - - //4. Scale everything so dVoxel==1 - Pfinal.x =Pfinal.x/geo.dVoxelX; Pfinal.y =Pfinal.y/geo.dVoxelY; Pfinal.z =Pfinal.z/geo.dVoxelZ; - Pfinalu0.x=Pfinalu0.x/geo.dVoxelX; Pfinalu0.y=Pfinalu0.y/geo.dVoxelY; Pfinalu0.z=Pfinalu0.z/geo.dVoxelZ; - Pfinalv0.x=Pfinalv0.x/geo.dVoxelX; Pfinalv0.y=Pfinalv0.y/geo.dVoxelY; Pfinalv0.z=Pfinalv0.z/geo.dVoxelZ; - S.x =S.x/geo.dVoxelX; S.y =S.y/geo.dVoxelY; S.z =S.z/geo.dVoxelZ; - - - - //5. apply COR. Wherever everything was, now its offesetd by a bit - float CORx, CORy; - CORx=-geo.COR[i]*sin(geo.alpha)/geo.dVoxelX; - CORy= geo.COR[i]*cos(geo.alpha)/geo.dVoxelY; - Pfinal.x+=CORx; Pfinal.y+=CORy; - Pfinalu0.x+=CORx; Pfinalu0.y+=CORy; - Pfinalv0.x+=CORx; Pfinalv0.y+=CORy; - S.x+=CORx; S.y+=CORy; - - // return - - *uvorigin=Pfinal; - - deltaU->x=Pfinalu0.x-Pfinal.x; - deltaU->y=Pfinalu0.y-Pfinal.y; - deltaU->z=Pfinalu0.z-Pfinal.z; - - deltaV->x=Pfinalv0.x-Pfinal.x; - deltaV->y=Pfinalv0.y-Pfinal.y; - deltaV->z=Pfinalv0.z-Pfinal.z; - - *source=S; -} -void CreateTextureParallelInterp(float* image,Geometry geo,cudaArray** d_cuArrTex, cudaTextureObject_t *texImage,cudaStream_t* stream){ //size_t size_image=geo.nVoxelX*geo.nVoxelY*geo.nVoxelZ; - - - const cudaExtent extent = make_cudaExtent(geo.nVoxelX, geo.nVoxelY, geo.nVoxelZ); - - //cudaArray Descriptor - cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(); - //cuda Array - cudaMalloc3DArray(&d_cuArrTex[0], &channelDesc, extent); - - - cudaMemcpy3DParms copyParams = {0}; - //Array creation - copyParams.srcPtr = make_cudaPitchedPtr((void *)image, extent.width*sizeof(float), extent.width, extent.height); - copyParams.dstArray = d_cuArrTex[0]; - copyParams.extent = extent; - copyParams.kind = cudaMemcpyHostToDevice; - cudaMemcpy3DAsync(©Params,stream[1]); - - - //Array creation End - - cudaResourceDesc texRes; - memset(&texRes, 0, sizeof(cudaResourceDesc)); - texRes.resType = cudaResourceTypeArray; - texRes.res.array.array = d_cuArrTex[0]; - cudaTextureDesc texDescr; - memset(&texDescr, 0, sizeof(cudaTextureDesc)); - texDescr.normalizedCoords = false; - texDescr.filterMode = cudaFilterModeLinear; - texDescr.addressMode[0] = cudaAddressModeBorder; - texDescr.addressMode[1] = cudaAddressModeBorder; - texDescr.addressMode[2] = cudaAddressModeBorder; - texDescr.readMode = cudaReadModeElementType; - cudaCreateTextureObject(&texImage[0], &texRes, &texDescr, NULL); - +/*------------------------------------------------------------------------- + * + * CUDA functions for texture-memory interpolation based projection + * + * This file has the necessary functions to perform X-ray parallel projection + * operation given a geaometry, angles and image. It uses the 3D texture + * memory linear interpolation to uniformily sample a path to integrate the + * X-rays. + * + * CODE by Ander Biguri + * Sepideh Hatamikia (arbitrary rotation) + * --------------------------------------------------------------------------- + * --------------------------------------------------------------------------- + * Copyright (c) 2015, University of Bath and CERN- European Organization for + * Nuclear Research + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * --------------------------------------------------------------------------- + * + * Contact: tigre.toolbox@gmail.com + * Codes : https://github.com/CERN/TIGRE + * --------------------------------------------------------------------------- + */ + + + +#include +#include +#include +#include "ray_interpolated_projection_parallel.hpp" +#include "TIGRE_common.hpp" +#include + +#define hipCheckErrors(msg) \ +do { \ + hipError_t __err = hipGetLastError(); \ + if (__err != hipSuccess) { \ + mexPrintf("%s \n",msg);\ + mexErrMsgIdAndTxt("TIGRE:Ax:interpolated_parallel",hipGetErrorString(__err));\ + } \ +} while (0) + + + +#define MAXTREADS 1024 +#define PROJ_PER_BLOCK 8 +#define PIXEL_SIZE_BLOCK 8 +/*GEOMETRY DEFINITION + * + * Detector plane, behind + * |-----------------------------| + * | | + * | | + * | | + * | | + * | +--------+ | + * | / /| | + * A Z | / / |*D | + * | | +--------+ | | + * | | | | | | + * | | | *O | + | + * --->y | | | / | + * / | | |/ | + * V X | +--------+ | + * |-----------------------------| + * + * *S + * + * + * + * + * + **/ +void CreateTextureParallelInterp(float* image,Geometry geo,hipArray** d_cuArrTex, hipTextureObject_t *texImage,hipStream_t* stream); +__constant__ Point3D projParamsArrayDev[4*PROJ_PER_BLOCK]; // Dev means it is on device +__constant__ float projFloatsArrayDev[2*PROJ_PER_BLOCK]; // Dev means it is on device + + + +__global__ void kernelPixelDetector_parallel_interpolated( Geometry geo, + float* detector, + const int currProjSetNumber, const int totalNoOfProjections, hipTextureObject_t tex) +{ +// Point3D source , +// Point3D deltaU, +// Point3D deltaV, +// Point3D uvOrigin, +// float DSO, +// float maxdist){ + + unsigned long long u = blockIdx.x * blockDim.x + threadIdx.x; + unsigned long long v = blockIdx.y * blockDim.y + threadIdx.y; + unsigned long long projNumber=threadIdx.z; + + if (u>= geo.nDetecU || v>= geo.nDetecV || projNumber>=PROJ_PER_BLOCK) + return; + + int indAlpha = currProjSetNumber*PROJ_PER_BLOCK+projNumber; // This is the ABSOLUTE projection number in the projection array + + +#if IS_FOR_MATLAB_TIGRE + size_t idx = (size_t)(u * (unsigned long long)geo.nDetecV + v)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; +#else + size_t idx = (size_t)(v * (unsigned long long)geo.nDetecU + u)+ projNumber*(unsigned long long)geo.nDetecV *(unsigned long long)geo.nDetecU ; +#endif + + if(indAlpha>=totalNoOfProjections) + return; + + Point3D uvOrigin = projParamsArrayDev[4*projNumber]; // 6*projNumber because we have 6 Point3D values per projection + Point3D deltaU = projParamsArrayDev[4*projNumber+1]; + Point3D deltaV = projParamsArrayDev[4*projNumber+2]; + Point3D source = projParamsArrayDev[4*projNumber+3]; + + float DSO = projFloatsArrayDev[2*projNumber+0]; + float maxdist = projFloatsArrayDev[2*projNumber+1]; + + + /////// Get coordinates XYZ of pixel UV + unsigned long pixelV = geo.nDetecV-v-1; + unsigned long pixelU = u; + + + float vectX,vectY,vectZ; + Point3D P; + P.x=(uvOrigin.x+pixelU*deltaU.x+pixelV*deltaV.x); + P.y=(uvOrigin.y+pixelU*deltaU.y+pixelV*deltaV.y); + P.z=(uvOrigin.z+pixelU*deltaU.z+pixelV*deltaV.z); + Point3D S; + S.x=(source.x+pixelU*deltaU.x+pixelV*deltaV.x); + S.y=(source.y+pixelU*deltaU.y+pixelV*deltaV.y); + S.z=(source.z+pixelU*deltaU.z+pixelV*deltaV.z); + + // Length is the ray length in normalized space + double length=sqrtf((S.x-P.x)*(S.x-P.x)+(S.y-P.y)*(S.y-P.y)+(S.z-P.z)*(S.z-P.z)); + //now legth is an integer of Nsamples that are required on this line + length=ceilf(length/geo.accuracy);//Divide the directional vector by an integer + vectX=(P.x -S.x)/(length); + vectY=(P.y -S.y)/(length); + vectZ=(P.z -S.z)/(length); + + +// //Integrate over the line + float tx,ty,tz; + float sum=0; + float i; + + + // limit the amount of mem access after the cube, but before the detector. + if ((2*DSO/geo.dVoxelX+maxdist)/geo.accuracy < length) + length=ceilf((2*DSO/geo.dVoxelX+maxdist)/geo.accuracy); + //Length is not actually a length, but the amount of memreads with given accuracy ("samples per voxel") + + for (i=floorf(maxdist/geo.accuracy); i<=length; i=i+1){ + tx=vectX*i+S.x; + ty=vectY*i+S.y; + tz=vectZ*i+S.z; + + sum += tex3D(tex, tx+0.5f, ty+0.5f, tz+0.5f); // this line is 94% of time. + + } + float deltalength=sqrtf((vectX*geo.dVoxelX)*(vectX*geo.dVoxelX)+ + (vectY*geo.dVoxelY)*(vectY*geo.dVoxelY)+ + (vectZ*geo.dVoxelZ)*(vectZ*geo.dVoxelZ) ); + detector[idx]=sum*deltalength; +} + + + +int interpolation_projection_parallel(float * img, Geometry geo, float** result,float const * const angles,int nangles, const GpuIds& gpuids){ + + + + size_t num_bytes = geo.nDetecU*geo.nDetecV *PROJ_PER_BLOCK* sizeof(float); + float** dProjection=(float **)malloc(2*sizeof(float *)); + for (int i = 0; i < 2; ++i){ + hipMalloc((void**)&dProjection[i], num_bytes); + hipCheckErrors("hipMalloc projections fail"); + } + // allocate streams for memory and compute + int nStreams=2; + hipStream_t* stream=(hipStream_t*)malloc(nStreams*sizeof(hipStream_t));; + + for (int i = 0; i < 2; ++i){ + hipStreamCreate(&stream[i]); + } + + + // Texture object variables + hipTextureObject_t *texImg = 0; + hipArray **d_cuArrTex = 0; + texImg =(hipTextureObject_t*)malloc(1*sizeof(hipTextureObject_t)); + d_cuArrTex =(hipArray**)malloc(1*sizeof(hipArray*)); + + CreateTextureParallelInterp(img,geo,&d_cuArrTex[0], &texImg[0],stream); + hipCheckErrors("Texture allocation fail"); + //Done! Image put into texture memory. + + + + Point3D source, deltaU, deltaV, uvOrigin; + + Point3D* projParamsArrayHost; + hipMallocHost((void**)&projParamsArrayHost,4*PROJ_PER_BLOCK*sizeof(Point3D)); + float* projFloatsArrayHost; + hipMallocHost((void**)&projFloatsArrayHost,2*PROJ_PER_BLOCK*sizeof(float)); + + // 16x16 gave the best performance empirically + // Funnily that makes it compatible with most GPUs..... + int divU,divV,divangle; + divU=PIXEL_SIZE_BLOCK; + divV=PIXEL_SIZE_BLOCK; + + dim3 numBlocks((geo.nDetecU+divU-1)/divU,(geo.nDetecV+divV-1)/divV,1); + dim3 threadsPerBlock(divU,divV,PROJ_PER_BLOCK); + unsigned int proj_global; + unsigned int noOfKernelCalls = (nangles+PROJ_PER_BLOCK-1)/PROJ_PER_BLOCK; // We'll take care of bounds checking inside the loop if nalpha is not divisible by PROJ_PER_BLOCK + unsigned int i; + + float maxdist; + for ( i=0; i=nangles) + break; + + geo.alpha=angles[proj_global*3]; + geo.theta=angles[proj_global*3+1]; + geo.psi =angles[proj_global*3+2]; + //precomute distances for faster execution + maxdist=maxdistanceCuboid(geo,proj_global); + //Precompute per angle constant stuff for speed + computeDeltas_parallel(geo,geo.alpha,proj_global, &uvOrigin, &deltaU, &deltaV, &source); + //Ray tracing! + projParamsArrayHost[4*j]=uvOrigin; // 6*j because we have 6 Point3D values per projection + projParamsArrayHost[4*j+1]=deltaU; + projParamsArrayHost[4*j+2]=deltaV; + projParamsArrayHost[4*j+3]=source; + + projFloatsArrayHost[2*j]=geo.DSO[proj_global]; + projFloatsArrayHost[2*j+1]=floor(maxdist); + + } + hipMemcpyToSymbolAsync(projParamsArrayDev, projParamsArrayHost, sizeof(Point3D)*4*PROJ_PER_BLOCK,0,hipMemcpyHostToDevice,stream[0]); + hipMemcpyToSymbolAsync(projFloatsArrayDev, projFloatsArrayHost, sizeof(float)*2*PROJ_PER_BLOCK,0,hipMemcpyHostToDevice,stream[0]); + hipStreamSynchronize(stream[0]); + + kernelPixelDetector_parallel_interpolated<<>>(geo,dProjection[(int)i%2==0],i,nangles,texImg[0]); + // copy result to host + if (i>0) + hipMemcpyAsync(result[i*PROJ_PER_BLOCK-PROJ_PER_BLOCK],dProjection[(int)i%2!=0], num_bytes, hipMemcpyDeviceToHost,stream[1]); + } + hipDeviceSynchronize(); + + int lastangles=nangles-(i-1)*PROJ_PER_BLOCK; + hipMemcpyAsync(result[(i-1)*PROJ_PER_BLOCK],dProjection[(int)(i-1)%2==0], lastangles*geo.nDetecV*geo.nDetecU*sizeof(float), hipMemcpyDeviceToHost,stream[1]); + + + hipDestroyTextureObject(texImg[0]); + hipFreeArray(d_cuArrTex[0]); + free(texImg); texImg = 0; + free(d_cuArrTex); d_cuArrTex = 0; + hipCheckErrors("Unbind fail"); + hipFree(dProjection[0]); + hipFree(dProjection[1]); + free(dProjection); + hipFreeHost(projParamsArrayHost); + hipFreeHost(projFloatsArrayHost); + + hipCheckErrors("hipFree d_imagedata fail"); + + + for (int i = 0; i < 2; ++i){ + hipStreamDestroy(stream[i]); + } +// hipDeviceReset(); + + return 0; +} + + + + +/* This code precomputes The location of the source and the Delta U and delta V (in the warped space) + * to compute the locations of the x-rays. While it seems verbose and overly-optimized, + * it does saves about 30% of each of the kernel calls. Thats something! + **/ +void computeDeltas_parallel(Geometry geo, float alpha,unsigned int i, Point3D* uvorigin, Point3D* deltaU, Point3D* deltaV, Point3D* source){ + Point3D S; + S.x=geo.DSO[i]; + S.y=geo.dDetecU*(0-((float)geo.nDetecU/2)+0.5); + S.z=geo.dDetecV*(((float)geo.nDetecV/2)-0.5-0); + + //End point + Point3D P,Pu0,Pv0; + + P.x =-(geo.DSD[i]-geo.DSO[i]); P.y = geo.dDetecU*(0-((float)geo.nDetecU/2)+0.5); P.z = geo.dDetecV*(((float)geo.nDetecV/2)-0.5-0); + Pu0.x=-(geo.DSD[i]-geo.DSO[i]); Pu0.y= geo.dDetecU*(1-((float)geo.nDetecU/2)+0.5); Pu0.z= geo.dDetecV*(((float)geo.nDetecV/2)-0.5-0); + Pv0.x=-(geo.DSD[i]-geo.DSO[i]); Pv0.y= geo.dDetecU*(0-((float)geo.nDetecU/2)+0.5); Pv0.z= geo.dDetecV*(((float)geo.nDetecV/2)-0.5-1); + // Geometric trasnformations: + P.x=0;Pu0.x=0;Pv0.x=0; + + // Roll pitch yaw + rollPitchYaw(geo,i,&P); + rollPitchYaw(geo,i,&Pu0); + rollPitchYaw(geo,i,&Pv0); + //Now lets translate the points where they should be: + P.x=P.x-(geo.DSD[i]-geo.DSO[i]); + Pu0.x=Pu0.x-(geo.DSD[i]-geo.DSO[i]); + Pv0.x=Pv0.x-(geo.DSD[i]-geo.DSO[i]); + + S.x=0; + // Roll pitch yaw + rollPitchYaw(geo,i,&S); + //Now lets translate the points where they should be: + S.x=S.x+geo.DSO[i]; + + + //1: Offset detector + + //P.x + P.y =P.y +geo.offDetecU[i]; P.z =P.z +geo.offDetecV[i]; + Pu0.y=Pu0.y+geo.offDetecU[i]; Pu0.z=Pu0.z+geo.offDetecV[i]; + Pv0.y=Pv0.y+geo.offDetecU[i]; Pv0.z=Pv0.z+geo.offDetecV[i]; + //S doesnt need to chagne + + + //3: Rotate (around z)! + Point3D Pfinal, Pfinalu0, Pfinalv0; + Pfinal.x =P.x; + Pfinal.y =P.y +geo.offDetecU[i]; Pfinal.z =P.z +geo.offDetecV[i]; + Pfinalu0.x=Pu0.x; + Pfinalu0.y=Pu0.y +geo.offDetecU[i]; Pfinalu0.z =Pu0.z +geo.offDetecV[i]; + Pfinalv0.x=Pv0.x; + Pfinalv0.y=Pv0.y +geo.offDetecU[i]; Pfinalv0.z =Pv0.z +geo.offDetecV[i]; + + eulerZYZ(geo,&Pfinal); + eulerZYZ(geo,&Pfinalu0); + eulerZYZ(geo,&Pfinalv0); + eulerZYZ(geo,&S); + + + + //2: Offset image (instead of offseting image, -offset everything else) + + Pfinal.x =Pfinal.x-geo.offOrigX[i]; Pfinal.y =Pfinal.y-geo.offOrigY[i]; Pfinal.z =Pfinal.z-geo.offOrigZ[i]; + Pfinalu0.x=Pfinalu0.x-geo.offOrigX[i]; Pfinalu0.y=Pfinalu0.y-geo.offOrigY[i]; Pfinalu0.z=Pfinalu0.z-geo.offOrigZ[i]; + Pfinalv0.x=Pfinalv0.x-geo.offOrigX[i]; Pfinalv0.y=Pfinalv0.y-geo.offOrigY[i]; Pfinalv0.z=Pfinalv0.z-geo.offOrigZ[i]; + S.x=S.x-geo.offOrigX[i]; S.y=S.y-geo.offOrigY[i]; S.z=S.z-geo.offOrigZ[i]; + + // As we want the (0,0,0) to be in a corner of the image, we need to translate everything (after rotation); + Pfinal.x =Pfinal.x+geo.sVoxelX/2-geo.dVoxelX/2; Pfinal.y =Pfinal.y+geo.sVoxelY/2-geo.dVoxelY/2; Pfinal.z =Pfinal.z +geo.sVoxelZ/2-geo.dVoxelZ/2; + Pfinalu0.x=Pfinalu0.x+geo.sVoxelX/2-geo.dVoxelX/2; Pfinalu0.y=Pfinalu0.y+geo.sVoxelY/2-geo.dVoxelY/2; Pfinalu0.z=Pfinalu0.z+geo.sVoxelZ/2-geo.dVoxelZ/2; + Pfinalv0.x=Pfinalv0.x+geo.sVoxelX/2-geo.dVoxelX/2; Pfinalv0.y=Pfinalv0.y+geo.sVoxelY/2-geo.dVoxelY/2; Pfinalv0.z=Pfinalv0.z+geo.sVoxelZ/2-geo.dVoxelZ/2; + S.x =S.x+geo.sVoxelX/2-geo.dVoxelX/2; S.y =S.y+geo.sVoxelY/2-geo.dVoxelY/2; S.z =S.z +geo.sVoxelZ/2-geo.dVoxelZ/2; + + //4. Scale everything so dVoxel==1 + Pfinal.x =Pfinal.x/geo.dVoxelX; Pfinal.y =Pfinal.y/geo.dVoxelY; Pfinal.z =Pfinal.z/geo.dVoxelZ; + Pfinalu0.x=Pfinalu0.x/geo.dVoxelX; Pfinalu0.y=Pfinalu0.y/geo.dVoxelY; Pfinalu0.z=Pfinalu0.z/geo.dVoxelZ; + Pfinalv0.x=Pfinalv0.x/geo.dVoxelX; Pfinalv0.y=Pfinalv0.y/geo.dVoxelY; Pfinalv0.z=Pfinalv0.z/geo.dVoxelZ; + S.x =S.x/geo.dVoxelX; S.y =S.y/geo.dVoxelY; S.z =S.z/geo.dVoxelZ; + + + + //5. apply COR. Wherever everything was, now its offesetd by a bit + float CORx, CORy; + CORx=-geo.COR[i]*sin(geo.alpha)/geo.dVoxelX; + CORy= geo.COR[i]*cos(geo.alpha)/geo.dVoxelY; + Pfinal.x+=CORx; Pfinal.y+=CORy; + Pfinalu0.x+=CORx; Pfinalu0.y+=CORy; + Pfinalv0.x+=CORx; Pfinalv0.y+=CORy; + S.x+=CORx; S.y+=CORy; + + // return + + *uvorigin=Pfinal; + + deltaU->x=Pfinalu0.x-Pfinal.x; + deltaU->y=Pfinalu0.y-Pfinal.y; + deltaU->z=Pfinalu0.z-Pfinal.z; + + deltaV->x=Pfinalv0.x-Pfinal.x; + deltaV->y=Pfinalv0.y-Pfinal.y; + deltaV->z=Pfinalv0.z-Pfinal.z; + + *source=S; +} +void CreateTextureParallelInterp(float* image,Geometry geo,hipArray** d_cuArrTex, hipTextureObject_t *texImage,hipStream_t* stream){ //size_t size_image=geo.nVoxelX*geo.nVoxelY*geo.nVoxelZ; + + + const hipExtent extent = make_cudaExtent(geo.nVoxelX, geo.nVoxelY, geo.nVoxelZ); + + //hipArray Descriptor + hipChannelFormatDesc channelDesc = hipCreateChannelDesc(); + //cuda Array + hipMalloc3DArray(&d_cuArrTex[0], &channelDesc, extent); + + + hipMemcpy3DParms copyParams = {0}; + //Array creation + copyParams.srcPtr = make_cudaPitchedPtr((void *)image, extent.width*sizeof(float), extent.width, extent.height); + copyParams.dstArray = d_cuArrTex[0]; + copyParams.extent = extent; + copyParams.kind = hipMemcpyHostToDevice; + hipMemcpy3DAsync(©Params,stream[1]); + + + //Array creation End + + hipResourceDesc texRes; + memset(&texRes, 0, sizeof(hipResourceDesc)); + texRes.resType = hipResourceTypeArray; + texRes.res.array.array = d_cuArrTex[0]; + hipTextureDesc texDescr; + memset(&texDescr, 0, sizeof(hipTextureDesc)); + texDescr.normalizedCoords = false; + texDescr.filterMode = hipFilterModeLinear; + texDescr.addressMode[0] = hipAddressModeBorder; + texDescr.addressMode[1] = hipAddressModeBorder; + texDescr.addressMode[2] = hipAddressModeBorder; + texDescr.readMode = hipReadModeElementType; + hipCreateTextureObject(&texImage[0], &texRes, &texDescr, NULL); + } \ No newline at end of file diff --git a/Common/CUDA/ray_interpolated_projection_parallel.hpp b/Common/CUDA/ray_interpolated_projection_parallel.hpp index 1280b6ed..c6ecf3e3 100644 --- a/Common/CUDA/ray_interpolated_projection_parallel.hpp +++ b/Common/CUDA/ray_interpolated_projection_parallel.hpp @@ -1,65 +1,65 @@ -/*------------------------------------------------------------------------- - * - * Header CUDA functions for texture-memory interpolation based projection - * - * - * CODE by Ander Biguri - * Sepideh Hatamikia (arbitrary rotation) ---------------------------------------------------------------------------- ---------------------------------------------------------------------------- -Copyright (c) 2015, University of Bath and CERN- European Organization for -Nuclear Research -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, -this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors -may be used to endorse or promote products derived from this software without -specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------- - -Contact: tigre.toolbox@gmail.com -Codes : https://github.com/CERN/TIGRE ---------------------------------------------------------------------------- - */ - - - - -#include "ray_interpolated_projection.hpp" - -#include "types_TIGRE.hpp" -#include "GpuIds.hpp" - -#ifndef PROJECTION_PARALLEL_HPP -#define PROJECTION_PARALLEL_HPP - -int interpolation_projection_parallel(float* img, Geometry geo, float** result,float const * const alphas,int nalpha, const GpuIds& gpuids); -// float computeMaxLength(Geometry geo, float alpha); -void computeDeltas_parallel(Geometry geo, float alpha,unsigned int i, Point3D* uvorigin, Point3D* deltaU, Point3D* deltaV, Point3D* source); - -// float maxDistanceCubeXY(Geometry geo, float alpha,int i); - -// below, not used -Geometry nomralizeGeometryImage(Geometry geo); +/*------------------------------------------------------------------------- + * + * Header CUDA functions for texture-memory interpolation based projection + * + * + * CODE by Ander Biguri + * Sepideh Hatamikia (arbitrary rotation) +--------------------------------------------------------------------------- +--------------------------------------------------------------------------- +Copyright (c) 2015, University of Bath and CERN- European Organization for +Nuclear Research +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + --------------------------------------------------------------------------- + +Contact: tigre.toolbox@gmail.com +Codes : https://github.com/CERN/TIGRE +--------------------------------------------------------------------------- + */ + + + + +#include "ray_interpolated_projection.hpp" + +#include "types_TIGRE.hpp" +#include "GpuIds.hpp" + +#ifndef PROJECTION_PARALLEL_HPP +#define PROJECTION_PARALLEL_HPP + +int interpolation_projection_parallel(float* img, Geometry geo, float** result,float const * const alphas,int nalpha, const GpuIds& gpuids); +// float computeMaxLength(Geometry geo, float alpha); +void computeDeltas_parallel(Geometry geo, float alpha,unsigned int i, Point3D* uvorigin, Point3D* deltaU, Point3D* deltaV, Point3D* source); + +// float maxDistanceCubeXY(Geometry geo, float alpha,int i); + +// below, not used +Geometry nomralizeGeometryImage(Geometry geo); #endif \ No newline at end of file diff --git a/Common/CUDA/tv_proximal.cu b/Common/CUDA/tv_proximal.cu index 102fd684..22d42209 100644 --- a/Common/CUDA/tv_proximal.cu +++ b/Common/CUDA/tv_proximal.cu @@ -1,693 +1,693 @@ -/*------------------------------------------------------------------------- - * - * MATLAB MEX functions for TV image denoising. Check inputs and parses - * MATLAB data to C++ data. - * - * - * CODE by Imanol Luengo - * PhD student University of Nottingham - * imaluengo@gmail.com - * 2015 - * Modified by Ander Biguri for multi-GPU - * --------------------------------------------------------------------------- - * --------------------------------------------------------------------------- - * Copyright (c) 2015, University of Bath and CERN- European Organization for - * Nuclear Research - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * --------------------------------------------------------------------------- - * - * Contact: tigre.toolbox@gmail.com - * Codes : https://github.com/CERN/TIGRE - * --------------------------------------------------------------------------- - */ - - - -// http://gpu4vision.icg.tugraz.at/papers/2010/knoll.pdf#pub47 -#define MAXTREADS 1024 -#define MAX_BUFFER 60 -#define BLOCK_SIZE 10 // BLOCK_SIZE^3 must be smaller than MAXTREADS - -#include "tv_proximal.hpp" -#define cudaCheckErrors(msg) \ -do { \ - cudaError_t __err = cudaGetLastError(); \ - if (__err != cudaSuccess) { \ - cudaDeviceReset();\ - mexPrintf("%s \n",msg);\ - mexErrMsgIdAndTxt("CBCT:CUDA:TVdenoising",cudaGetErrorString(__err));\ - } \ -} while (0) -void cpy_from_host(float* device_array,float* host_array, - unsigned long long bytes_device,unsigned long long offset_device,unsigned long long offset_host, - unsigned long long pixels_per_slice, unsigned int buffer_length, - cudaStream_t stream, bool is_first_chunk, bool is_last_chunk,const long* image_size); - - - __global__ void multiplyArrayScalar(float* vec,float scalar,const size_t n) - { - unsigned long long i = (blockIdx.x * blockDim.x) + threadIdx.x; - for(; i= 0 ) { - _div += (pz[idx] - pz[(z-1)*size2d + y*cols + x]) / dz; - } else { - _div += pz[idx]; - } - - if ( y - 1 >= 0 ) { - _div += (py[idx] - py[z*size2d + (y-1)*cols + x]) / dy; - } else { - _div += py[idx]; - } - - if ( x - 1 >= 0 ) { - _div += (px[idx] - px[z*size2d + y*cols + (x-1)]) / dx; - } else { - _div += px[idx]; - } - - return _div; - } - - __device__ __inline__ - void gradient(const float* u, float* grad, - long z, long y, long x, - long depth, long rows, long cols, - float dz, float dy, float dx) - { - long size2d = rows*cols; - long idx = z * size2d + y * cols + x; - - float uidx = u[idx]; - - if ( z + 1 < depth ) { - grad[0] = (u[(z+1)*size2d + y*cols + x] - uidx) / dz; - } - - if ( y + 1 < rows ) { - grad[1] = (u[z*size2d + (y+1)*cols + x] - uidx) / dy; - } - - if ( x + 1 < cols ) { - grad[2] = (u[z*size2d + y*cols + (x+1)] - uidx) / dx; - } - } - - - __global__ - void update_u(const float* f, const float* pz, const float* py, const float* px, float* u, - float tau, float lambda, - long depth, long rows, long cols, - float dz, float dy, float dx) - { - long x = threadIdx.x + blockIdx.x * blockDim.x; - long y = threadIdx.y + blockIdx.y * blockDim.y; - long z = threadIdx.z + blockIdx.z * blockDim.z; - long idx = z * rows * cols + y * cols + x; - - if ( x >= cols || y >= rows || z >= depth ) - return; - - float _div = divergence(pz, py, px, z, y, x, depth, rows, cols, dz, dy, dx); - - u[idx] = u[idx] * (1.0f - tau) + tau * (f[idx] + (1.0f/lambda) * _div); - } - - - __global__ - void update_p(const float* u, float* pz, float* py, float* px, - float tau, long depth, long rows, long cols, - float dz, float dy, float dx) - { - long x = threadIdx.x + blockIdx.x * blockDim.x; - long y = threadIdx.y + blockIdx.y * blockDim.y; - long z = threadIdx.z + blockIdx.z * blockDim.z; - long idx = z * rows * cols + y * cols + x; - - if ( x >= cols || y >= rows || z >= depth ) - return; - - float grad[3] = {0,0,0}, q[3]; - gradient(u, grad, z, y, x, depth, rows, cols, dz, dy, dx); - - q[0] = pz[idx] + tau * grad[0]; - q[1] = py[idx] + tau * grad[1]; - q[2] = px[idx] + tau * grad[2]; - - float norm = fmaxf(1.0f, sqrtf(q[0] * q[0] + q[1] * q[1] + q[2] * q[2])); - - pz[idx] = q[0] / norm; - py[idx] = q[1] / norm; - px[idx] = q[2] / norm; - } - - -// Main function - void tvdenoising(float* src, float* dst, float lambda, - const float* spacing, const long* image_size, int maxIter, const GpuIds& gpuids) { - - // Prepare for MultiGPU - int deviceCount = gpuids.GetLength(); - cudaCheckErrors("Device query fail"); - if (deviceCount == 0) { - mexErrMsgIdAndTxt("tvDenoise:tvdenoising:GPUselect","There are no available device(s) that support CUDA\n"); - } - // - // CODE assumes - // 1.-All available devices are usable by this code - // 2.-All available devices are equal, they are the same machine (warning thrown) - // Check the available devices, and if they are the same - if (!gpuids.AreEqualDevices()) { - mexWarnMsgIdAndTxt("tvDenoise:tvdenoising:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); - } - int dev; - - // We don't know if the devices are being used. lets check that. and only use the amount of memory we need. - - size_t mem_GPU_global; - checkFreeMemory(gpuids, &mem_GPU_global); - - - // %5 of free memory should be enough, we have almost no variables in these kernels - size_t total_pixels = image_size[0] * image_size[1] * image_size[2] ; - const size_t pixels_per_slice = image_size[0] * image_size[1] ; - const size_t mem_slice_image = sizeof(float)* pixels_per_slice ; - const size_t mem_size_image = sizeof(float)* total_pixels; - - // Decide how are we handling the distribution of computation - size_t mem_img_each_GPU; - - unsigned int buffer_length=1; - //Does everything fit in the GPU? - unsigned int slices_per_split; - unsigned int splits=1; // if the number does not fit in an uint, you have more serious trouble than this. - if(mem_GPU_global> 5*mem_size_image+5*mem_slice_image*buffer_length*2){ - // We only need to split if we have extra GPUs - slices_per_split=(image_size[2]+deviceCount-1)/deviceCount; - mem_img_each_GPU=mem_slice_image*( (image_size[2]+deviceCount-1)/deviceCount + buffer_length*2); - }else{ - // As mem_auxiliary is not expected to be a large value (for a 2000^3 image is around 28Mbytes), lets for now assume we need it all - size_t mem_free=mem_GPU_global; - - splits=(unsigned int)(ceil(((float)(5*mem_size_image)/(float)(deviceCount))/mem_free)); - // Now, there is an overhead here, as each splits should have 2 slices more, to account for overlap of images. - // lets make sure these 2 slices fit, if they do not, add 1 to splits. - slices_per_split=(image_size[2]+deviceCount*splits-1)/(deviceCount*splits); - mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); - - // if the new stuff does not fit in the GPU, it means we are in the edge case where adding that extra slice will overflow memory - if (mem_GPU_global< 5*mem_img_each_GPU){ - // one more split should do the job, as its an edge case. - splits++; - //recompute for later - slices_per_split=(image_size[2]+deviceCount*splits-1)/(deviceCount*splits); // amount of slices that fit on a GPU. Later we add 2 to these, as we need them for overlap - mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); - } - - // How many EXTRA buffer slices should be able to fit in here??!?! - mem_free=mem_GPU_global-(5*mem_img_each_GPU); - unsigned int extra_buff=(mem_free/mem_slice_image); - buffer_length=(extra_buff/2)/5; // we need double whatever this results in, rounded down. - - buffer_length=min(MAX_BUFFER,buffer_length); - - mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); - - // Assert - if (mem_GPU_global< 5*mem_img_each_GPU){ - mexErrMsgIdAndTxt("tvDenoise:tvdenoising:GPU","Bad assert. Logic behind splitting flawed! Please tell: ander.biguri@gmail.com\n"); - } - } - - - // Lets try to make the host memory pinned: - // We laredy queried the GPU and assuemd they are the same, thus should have the same attributes. - int isHostRegisterSupported = 0; -#if CUDART_VERSION >= 9020 - cudaDeviceGetAttribute(&isHostRegisterSupported,cudaDevAttrHostRegisterSupported,gpuids[0]); -#endif - if (isHostRegisterSupported & splits>1){ - cudaHostRegister(src ,image_size[2]*image_size[1]*image_size[0]*sizeof(float),cudaHostRegisterPortable); - cudaHostRegister(dst ,image_size[2]*image_size[1]*image_size[0]*sizeof(float),cudaHostRegisterPortable); - } - cudaCheckErrors("Error pinning memory"); - - - - // Lets allocate auxiliary variables. - float* buffer_u, *buffer_px, *buffer_py, *buffer_pz; - float* h_px, *h_py, *h_pz, *h_u; - if(splits>1){ - - //These take A LOT of memory and A LOT of time to use. If we can avoid using them, better. - if (buffer_length1 & i>0){ - - for (dev = 0; dev < deviceCount; dev++){ - is_last_chunk=!((sp*deviceCount+dev)>>(d_pz[dev], -1, pixels_per_slice*buffer_length); - } - if (is_last_chunk){ - multiplyArrayScalar<<<60,MAXTREADS,0,stream[dev*nStream_device+4]>>>(d_pz[dev]+bytes_device[dev],-1, pixels_per_slice*buffer_length); - } - } - for (dev = 0; dev < deviceCount; dev++){ - is_last_chunk=!((sp*deviceCount+dev)>>(d_src[dev], d_pz[dev], d_py[dev], d_px[dev], d_u[dev], tau1, lambda, - (long)(curr_slices+buffer_length*2), image_size[1],image_size[0], - spacing[2], spacing[1], spacing[0]); - } - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - curr_slices=((sp*deviceCount+dev+1)*slices_per_split>>(d_u[dev], d_pz[dev], d_py[dev], d_px[dev], tau2, - (long)(curr_slices+buffer_length*2), image_size[1], image_size[0], - spacing[2], spacing[1], spacing[0]); - } - }// END internal iter - - // Synchronize mathematics, make sure bounding pixels are correct - for(dev=0; dev0){ - // U - cudaSetDevice(gpuids[dev-1]); - cudaMemcpyAsync(buffer_u, d_u[dev-1] +slices_per_split*pixels_per_slice+buffer_pixels, buffer_pixels*sizeof(float), cudaMemcpyDeviceToHost,stream[(dev-1)*nStream_device+1]); - cudaMemcpyAsync(buffer_px, d_px[dev-1]+slices_per_split*pixels_per_slice+buffer_pixels, buffer_pixels*sizeof(float), cudaMemcpyDeviceToHost,stream[(dev-1)*nStream_device+2]); - cudaMemcpyAsync(buffer_py, d_py[dev-1]+slices_per_split*pixels_per_slice+buffer_pixels, buffer_pixels*sizeof(float), cudaMemcpyDeviceToHost,stream[(dev-1)*nStream_device+3]); - cudaMemcpyAsync(buffer_pz, d_pz[dev-1]+slices_per_split*pixels_per_slice+buffer_pixels, buffer_pixels*sizeof(float), cudaMemcpyDeviceToHost,stream[(dev-1)*nStream_device+4]); - - - cudaSetDevice(gpuids[dev]); - cudaStreamSynchronize(stream[(dev-1)*nStream_device+1]); - cudaMemcpyAsync(d_u[dev] ,buffer_u , buffer_pixels*sizeof(float), cudaMemcpyHostToDevice,stream[(dev)*nStream_device+1]); - cudaStreamSynchronize(stream[(dev-1)*nStream_device+2]); - cudaMemcpyAsync(d_px[dev],buffer_px, buffer_pixels*sizeof(float), cudaMemcpyHostToDevice,stream[(dev)*nStream_device+2]); - cudaStreamSynchronize(stream[(dev-1)*nStream_device+3]); - cudaMemcpyAsync(d_py[dev],buffer_py, buffer_pixels*sizeof(float), cudaMemcpyHostToDevice,stream[(dev)*nStream_device+3]); - cudaStreamSynchronize(stream[(dev-1)*nStream_device+4]); - cudaMemcpyAsync(d_pz[dev],buffer_pz, buffer_pixels*sizeof(float), cudaMemcpyHostToDevice,stream[(dev)*nStream_device+4]); - - - } - } - // This is the case when we can't solely use GPU memory, as the total size of the images+variables exceeds total amounf of memory among GPUs. - // This situation requires partial results and full memory allocation in the host. - }else{ - // Vopy all the U variable into the host. - for(dev=0; dev1 && buffer_length1){ - cudaHostUnregister(src); - cudaHostUnregister(dst); - } - for(dev=0; dev 1){ - for (unsigned int j=0;j 1){ - - for (unsigned int j=0;j= 0 ) { + _div += (pz[idx] - pz[(z-1)*size2d + y*cols + x]) / dz; + } else { + _div += pz[idx]; + } + + if ( y - 1 >= 0 ) { + _div += (py[idx] - py[z*size2d + (y-1)*cols + x]) / dy; + } else { + _div += py[idx]; + } + + if ( x - 1 >= 0 ) { + _div += (px[idx] - px[z*size2d + y*cols + (x-1)]) / dx; + } else { + _div += px[idx]; + } + + return _div; + } + + __device__ __inline__ + void gradient(const float* u, float* grad, + long z, long y, long x, + long depth, long rows, long cols, + float dz, float dy, float dx) + { + long size2d = rows*cols; + long idx = z * size2d + y * cols + x; + + float uidx = u[idx]; + + if ( z + 1 < depth ) { + grad[0] = (u[(z+1)*size2d + y*cols + x] - uidx) / dz; + } + + if ( y + 1 < rows ) { + grad[1] = (u[z*size2d + (y+1)*cols + x] - uidx) / dy; + } + + if ( x + 1 < cols ) { + grad[2] = (u[z*size2d + y*cols + (x+1)] - uidx) / dx; + } + } + + + __global__ + void update_u(const float* f, const float* pz, const float* py, const float* px, float* u, + float tau, float lambda, + long depth, long rows, long cols, + float dz, float dy, float dx) + { + long x = threadIdx.x + blockIdx.x * blockDim.x; + long y = threadIdx.y + blockIdx.y * blockDim.y; + long z = threadIdx.z + blockIdx.z * blockDim.z; + long idx = z * rows * cols + y * cols + x; + + if ( x >= cols || y >= rows || z >= depth ) + return; + + float _div = divergence(pz, py, px, z, y, x, depth, rows, cols, dz, dy, dx); + + u[idx] = u[idx] * (1.0f - tau) + tau * (f[idx] + (1.0f/lambda) * _div); + } + + + __global__ + void update_p(const float* u, float* pz, float* py, float* px, + float tau, long depth, long rows, long cols, + float dz, float dy, float dx) + { + long x = threadIdx.x + blockIdx.x * blockDim.x; + long y = threadIdx.y + blockIdx.y * blockDim.y; + long z = threadIdx.z + blockIdx.z * blockDim.z; + long idx = z * rows * cols + y * cols + x; + + if ( x >= cols || y >= rows || z >= depth ) + return; + + float grad[3] = {0,0,0}, q[3]; + gradient(u, grad, z, y, x, depth, rows, cols, dz, dy, dx); + + q[0] = pz[idx] + tau * grad[0]; + q[1] = py[idx] + tau * grad[1]; + q[2] = px[idx] + tau * grad[2]; + + float norm = fmaxf(1.0f, sqrtf(q[0] * q[0] + q[1] * q[1] + q[2] * q[2])); + + pz[idx] = q[0] / norm; + py[idx] = q[1] / norm; + px[idx] = q[2] / norm; + } + + +// Main function + void tvdenoising(float* src, float* dst, float lambda, + const float* spacing, const long* image_size, int maxIter, const GpuIds& gpuids) { + + // Prepare for MultiGPU + int deviceCount = gpuids.GetLength(); + hipCheckErrors("Device query fail"); + if (deviceCount == 0) { + mexErrMsgIdAndTxt("tvDenoise:tvdenoising:GPUselect","There are no available device(s) that support CUDA\n"); + } + // + // CODE assumes + // 1.-All available devices are usable by this code + // 2.-All available devices are equal, they are the same machine (warning thrown) + // Check the available devices, and if they are the same + if (!gpuids.AreEqualDevices()) { + mexWarnMsgIdAndTxt("tvDenoise:tvdenoising:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); + } + int dev; + + // We don't know if the devices are being used. lets check that. and only use the amount of memory we need. + + size_t mem_GPU_global; + checkFreeMemory(gpuids, &mem_GPU_global); + + + // %5 of free memory should be enough, we have almost no variables in these kernels + size_t total_pixels = image_size[0] * image_size[1] * image_size[2] ; + const size_t pixels_per_slice = image_size[0] * image_size[1] ; + const size_t mem_slice_image = sizeof(float)* pixels_per_slice ; + const size_t mem_size_image = sizeof(float)* total_pixels; + + // Decide how are we handling the distribution of computation + size_t mem_img_each_GPU; + + unsigned int buffer_length=1; + //Does everything fit in the GPU? + unsigned int slices_per_split; + unsigned int splits=1; // if the number does not fit in an uint, you have more serious trouble than this. + if(mem_GPU_global> 5*mem_size_image+5*mem_slice_image*buffer_length*2){ + // We only need to split if we have extra GPUs + slices_per_split=(image_size[2]+deviceCount-1)/deviceCount; + mem_img_each_GPU=mem_slice_image*( (image_size[2]+deviceCount-1)/deviceCount + buffer_length*2); + }else{ + // As mem_auxiliary is not expected to be a large value (for a 2000^3 image is around 28Mbytes), lets for now assume we need it all + size_t mem_free=mem_GPU_global; + + splits=(unsigned int)(ceil(((float)(5*mem_size_image)/(float)(deviceCount))/mem_free)); + // Now, there is an overhead here, as each splits should have 2 slices more, to account for overlap of images. + // lets make sure these 2 slices fit, if they do not, add 1 to splits. + slices_per_split=(image_size[2]+deviceCount*splits-1)/(deviceCount*splits); + mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); + + // if the new stuff does not fit in the GPU, it means we are in the edge case where adding that extra slice will overflow memory + if (mem_GPU_global< 5*mem_img_each_GPU){ + // one more split should do the job, as its an edge case. + splits++; + //recompute for later + slices_per_split=(image_size[2]+deviceCount*splits-1)/(deviceCount*splits); // amount of slices that fit on a GPU. Later we add 2 to these, as we need them for overlap + mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); + } + + // How many EXTRA buffer slices should be able to fit in here??!?! + mem_free=mem_GPU_global-(5*mem_img_each_GPU); + unsigned int extra_buff=(mem_free/mem_slice_image); + buffer_length=(extra_buff/2)/5; // we need double whatever this results in, rounded down. + + buffer_length=min(MAX_BUFFER,buffer_length); + + mem_img_each_GPU=(mem_slice_image*(slices_per_split+buffer_length*2)); + + // Assert + if (mem_GPU_global< 5*mem_img_each_GPU){ + mexErrMsgIdAndTxt("tvDenoise:tvdenoising:GPU","Bad assert. Logic behind splitting flawed! Please tell: ander.biguri@gmail.com\n"); + } + } + + + // Lets try to make the host memory pinned: + // We laredy queried the GPU and assuemd they are the same, thus should have the same attributes. + int isHostRegisterSupported = 0; +#if CUDART_VERSION >= 9020 + hipDeviceGetAttribute(&isHostRegisterSupported,hipDevAttrHostRegisterSupported,gpuids[0]); +#endif + if (isHostRegisterSupported & splits>1){ + hipHostRegister(src ,image_size[2]*image_size[1]*image_size[0]*sizeof(float),hipHostRegisterPortable); + hipHostRegister(dst ,image_size[2]*image_size[1]*image_size[0]*sizeof(float),hipHostRegisterPortable); + } + hipCheckErrors("Error pinning memory"); + + + + // Lets allocate auxiliary variables. + float* buffer_u, *buffer_px, *buffer_py, *buffer_pz; + float* h_px, *h_py, *h_pz, *h_u; + if(splits>1){ + + //These take A LOT of memory and A LOT of time to use. If we can avoid using them, better. + if (buffer_length1 & i>0){ + + for (dev = 0; dev < deviceCount; dev++){ + is_last_chunk=!((sp*deviceCount+dev)>>(d_pz[dev], -1, pixels_per_slice*buffer_length); + } + if (is_last_chunk){ + multiplyArrayScalar<<<60,MAXTREADS,0,stream[dev*nStream_device+4]>>>(d_pz[dev]+bytes_device[dev],-1, pixels_per_slice*buffer_length); + } + } + for (dev = 0; dev < deviceCount; dev++){ + is_last_chunk=!((sp*deviceCount+dev)>>(d_src[dev], d_pz[dev], d_py[dev], d_px[dev], d_u[dev], tau1, lambda, + (long)(curr_slices+buffer_length*2), image_size[1],image_size[0], + spacing[2], spacing[1], spacing[0]); + } + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + curr_slices=((sp*deviceCount+dev+1)*slices_per_split>>(d_u[dev], d_pz[dev], d_py[dev], d_px[dev], tau2, + (long)(curr_slices+buffer_length*2), image_size[1], image_size[0], + spacing[2], spacing[1], spacing[0]); + } + }// END internal iter + + // Synchronize mathematics, make sure bounding pixels are correct + for(dev=0; dev0){ + // U + hipSetDevice(gpuids[dev-1]); + hipMemcpyAsync(buffer_u, d_u[dev-1] +slices_per_split*pixels_per_slice+buffer_pixels, buffer_pixels*sizeof(float), hipMemcpyDeviceToHost,stream[(dev-1)*nStream_device+1]); + hipMemcpyAsync(buffer_px, d_px[dev-1]+slices_per_split*pixels_per_slice+buffer_pixels, buffer_pixels*sizeof(float), hipMemcpyDeviceToHost,stream[(dev-1)*nStream_device+2]); + hipMemcpyAsync(buffer_py, d_py[dev-1]+slices_per_split*pixels_per_slice+buffer_pixels, buffer_pixels*sizeof(float), hipMemcpyDeviceToHost,stream[(dev-1)*nStream_device+3]); + hipMemcpyAsync(buffer_pz, d_pz[dev-1]+slices_per_split*pixels_per_slice+buffer_pixels, buffer_pixels*sizeof(float), hipMemcpyDeviceToHost,stream[(dev-1)*nStream_device+4]); + + + hipSetDevice(gpuids[dev]); + hipStreamSynchronize(stream[(dev-1)*nStream_device+1]); + hipMemcpyAsync(d_u[dev] ,buffer_u , buffer_pixels*sizeof(float), hipMemcpyHostToDevice,stream[(dev)*nStream_device+1]); + hipStreamSynchronize(stream[(dev-1)*nStream_device+2]); + hipMemcpyAsync(d_px[dev],buffer_px, buffer_pixels*sizeof(float), hipMemcpyHostToDevice,stream[(dev)*nStream_device+2]); + hipStreamSynchronize(stream[(dev-1)*nStream_device+3]); + hipMemcpyAsync(d_py[dev],buffer_py, buffer_pixels*sizeof(float), hipMemcpyHostToDevice,stream[(dev)*nStream_device+3]); + hipStreamSynchronize(stream[(dev-1)*nStream_device+4]); + hipMemcpyAsync(d_pz[dev],buffer_pz, buffer_pixels*sizeof(float), hipMemcpyHostToDevice,stream[(dev)*nStream_device+4]); + + + } + } + // This is the case when we can't solely use GPU memory, as the total size of the images+variables exceeds total amounf of memory among GPUs. + // This situation requires partial results and full memory allocation in the host. + }else{ + // Vopy all the U variable into the host. + for(dev=0; dev1 && buffer_length1){ + hipHostUnregister(src); + hipHostUnregister(dst); + } + for(dev=0; dev 1){ + for (unsigned int j=0;j 1){ + + for (unsigned int j=0;j Origin is at (0,0,0). Image center is there +offOrig - // -> at angle 0, source + image centre (without the offset) + detector centre (without offset) - // are aligned in the Y_Z plane. - // -> detector is orthonormal to projection plane. - - //Parameters part of the image geometry - int nVoxelX, nVoxelY, nVoxelZ; - float sVoxelX, sVoxelY, sVoxelZ; - float dVoxelX, dVoxelY, dVoxelZ; - float *offOrigX,*offOrigY,*offOrigZ; - float* DSO; - // Parameters of the Detector. - int nDetecU, nDetecV; - float sDetecU, sDetecV; - float dDetecU, dDetecV; - float *offDetecU, *offDetecV; - float* DSD; - float* dRoll; - float* dPitch; - float* dYaw; - // The base unit we are working with in mm. - float unitX; - float unitY; - float unitZ; - - //rotation angle for e uler (ZYZ) - float alpha; - float theta; - float psi; - // Centre of Rotation correction. - float* COR; - //Maximum length of cube - float maxLength; - //User option - float accuracy; -}; - - struct Point3D{ - float x; - float y; - float z; -}; - -struct Point3Ddouble{ - double x; - double y; - double z; - - // cast to float member function for "copying" Point3Ddouble to Point3D - Point3D to_float() - { - Point3D castToFloat; - castToFloat.x = (float)x; - castToFloat.y = (float)y; - castToFloat.z = (float)z; - return(castToFloat); - } -}; - +/*------------------------------------------------------------------------- + * + * Header CUDA functions for texture-memory interpolation based projection + * + * + * CODE by Ander Biguri + * Sepideh Hatamikia (arbitrary rotation) +--------------------------------------------------------------------------- +--------------------------------------------------------------------------- +Copyright (c) 2015, University of Bath and CERN- European Organization for +Nuclear Research +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + --------------------------------------------------------------------------- + +Contact: tigre.toolbox@gmail.com +Codes : https://github.com/CERN/TIGRE +--------------------------------------------------------------------------- + */ + +#ifndef TYPES_CBCT +#define TYPES_CBCT +struct Geometry { + // Geometry assumptions: + // -> Origin is at (0,0,0). Image center is there +offOrig + // -> at angle 0, source + image centre (without the offset) + detector centre (without offset) + // are aligned in the Y_Z plane. + // -> detector is orthonormal to projection plane. + + //Parameters part of the image geometry + int nVoxelX, nVoxelY, nVoxelZ; + float sVoxelX, sVoxelY, sVoxelZ; + float dVoxelX, dVoxelY, dVoxelZ; + float *offOrigX,*offOrigY,*offOrigZ; + float* DSO; + // Parameters of the Detector. + int nDetecU, nDetecV; + float sDetecU, sDetecV; + float dDetecU, dDetecV; + float *offDetecU, *offDetecV; + float* DSD; + float* dRoll; + float* dPitch; + float* dYaw; + // The base unit we are working with in mm. + float unitX; + float unitY; + float unitZ; + + //rotation angle for e uler (ZYZ) + float alpha; + float theta; + float psi; + // Centre of Rotation correction. + float* COR; + //Maximum length of cube + float maxLength; + //User option + float accuracy; +}; + + struct Point3D{ + float x; + float y; + float z; +}; + +struct Point3Ddouble{ + double x; + double y; + double z; + + // cast to float member function for "copying" Point3Ddouble to Point3D + Point3D to_float() + { + Point3D castToFloat; + castToFloat.x = (float)x; + castToFloat.y = (float)y; + castToFloat.z = (float)z; + return(castToFloat); + } +}; + #endif \ No newline at end of file diff --git a/Common/CUDA/voxel_backprojection.cu b/Common/CUDA/voxel_backprojection.cu index bec4d909..c7c931ae 100644 --- a/Common/CUDA/voxel_backprojection.cu +++ b/Common/CUDA/voxel_backprojection.cu @@ -1,920 +1,920 @@ -/*------------------------------------------------------------------------- - * - * CUDA function for backrpojection using FDK weigts for CBCT - * - * - * CODE by Ander Biguri - * Optimized and modified by RB - * --------------------------------------------------------------------------- - * --------------------------------------------------------------------------- - * Copyright (c) 2015, University of Bath and CERN- European Organization for - * Nuclear Research - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its contributors - * may be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * --------------------------------------------------------------------------- - * - * Contact: tigre.toolbox@gmail.com - * Codes : https://github.com/CERN/TIGRE - * --------------------------------------------------------------------------- - */ - -#define PI_2 1.57079632679489661923 -#include -#include -#include -#include "voxel_backprojection.hpp" -#include "TIGRE_common.hpp" -#include -#include "GpuIds.hpp" - -// https://stackoverflow.com/questions/16282136/is-there-a-cuda-equivalent-of-perror -#define cudaCheckErrors(msg) \ -do { \ - cudaError_t __err = cudaGetLastError(); \ - if (__err != cudaSuccess) { \ - mexPrintf("%s \n",msg);\ - mexErrMsgIdAndTxt("CBCT:CUDA:Atb",cudaGetErrorString(__err));\ - } \ -} while (0) - - -#define MAXTREADS 1024 - /*GEOMETRY DEFINITION - * - * Detector plane, behind - * |-----------------------------| - * | | - * | | - * | | - * | | - * | +--------+ | - * | / /| | - * A Z | / / |*D | - * | | +--------+ | | - * | | | | | | - * | | | *O | + | - * *--->y | | | / | - * / | | |/ | - * V X | +--------+ | - * |-----------------------------| - * - * *S - * - * - * - * - * - **/ - - void CreateTexture(const GpuIds& gpuids,float* projectiondata,Geometry geo,cudaArray** d_cuArrTex,unsigned int nangles, cudaTextureObject_t *texImage,cudaStream_t* stream, int nStreamDevice,bool allocate); - - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// RB, 10/31/2016: Add constant memory arrays to store parameters for all projections to be analyzed during a single kernel call -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -// The optimal values of two constants obtained by RB on NVIDIA Quadro K2200 (4 GB RAM, 640 CUDA cores) for 512^3 volume and 512^3 projections (512 proj, each 512 x 512) were: -// PROJ_PER_KERNEL = 32 or 16 (very similar times) -// VOXELS_PER_THREAD = 8 -// Speedup of the entire FDK backprojection (not only kernel run, also memcpy etc.) was nearly 4x relative to the original (single projection, single voxel per thread) code. -// (e.g. 16.2 s vs. ~62 s). - -const int PROJ_PER_KERNEL = 32; // Number of 2D projections to be analyzed by a single thread. This can be tweaked to see what works best. 32 was the optimal value in the paper by Zinsser and Keck. -const int VOXELS_PER_THREAD = 8; // Number of voxels to be computed by s single thread. Can be tweaked to see what works best. 4 was the optimal value in the paper by Zinsser and Keck. - -// We have PROJ_PER_KERNEL projections and we need 6 parameters for each projection: -// deltaX, deltaY, deltaZ, xyzOrigin, offOrig, offDetec -// So we need to keep PROJ_PER_KERNEL*6 values in our deltas array FOR EACH CALL to our main kernel -// (they will be updated in the main loop before each kernel call). - -__constant__ Point3D projParamsArrayDev[6*PROJ_PER_KERNEL]; // Dev means it is on device - -// We also need a corresponding array on the host side to be filled before each kernel call, then copied to the device (array in constant memory above) -// Point3D projParamsArrayHost[6*PROJ_PER_KERNEL]; // Host means it is host memory - -// Now we also need to store sinAlpha and cosAlpha for each projection (two floats per projection) -__constant__ float projSinCosArrayDev[5*PROJ_PER_KERNEL]; - - - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// END RB, 10/31/2016: Add constant memory arrays to store parameters for all projections to be analyzed during a single kernel call -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - -//______________________________________________________________________________ -// -// Function: kernelPixelBackprojectionFDK -// -// Description: Main FDK backprojection kernel -//______________________________________________________________________________ - -__global__ void kernelPixelBackprojectionFDK(const Geometry geo, float* image,const int currProjSetNumber, const int totalNoOfProjections, cudaTextureObject_t tex) -{ - - // Old kernel call signature: - // kernelPixelBackprojectionFDK<<>>(geo,dimage,i,deltaX,deltaY,deltaZ,xyzOrigin,offOrig,offDetec,sinalpha,cosalpha); - // We just read in most of the params from the constant memory instead of getting them from the param list. - // This is because we now have MANY params, since single kernel processes more than one projection! - /* __global__ void kernelPixelBackprojectionFDK(const Geometry geo, - * float* image, - * const int indAlpha, - * const Point3D deltaX , - * const Point3D deltaY, - * const Point3D deltaZ, - * const Point3D xyzOrigin, - * const Point3D xyzOffset, - * const Point3D uv0Offset, - * const float sinalpha, - * const float cosalpha){ - */ - unsigned long long indY = blockIdx.y * blockDim.y + threadIdx.y; - unsigned long long indX = blockIdx.x * blockDim.x + threadIdx.x; - // unsigned long startIndZ = blockIdx.z * blockDim.z + threadIdx.z; // This is only STARTING z index of the column of voxels that the thread will handle - unsigned long long startIndZ = blockIdx.z * VOXELS_PER_THREAD + threadIdx.z; // This is only STARTING z index of the column of voxels that the thread will handle - //Make sure we don't go out of bounds - if (indX>=geo.nVoxelX || indY>=geo.nVoxelY || startIndZ>=geo.nVoxelZ) - return; - - // We'll keep a local auxiliary array of values of a column of voxels that this thread will update - float voxelColumn[VOXELS_PER_THREAD]; - - // First we need to copy the curent 3D volume values from the column to our auxiliary array so that we can then - // work on them (update them by computing values from multiple projections) locally - avoiding main memory reads/writes - - unsigned long colIdx; -#pragma unroll - for(colIdx=0; colIdx=geo.nVoxelZ) - break; // break the loop. - - unsigned long long idx =indZ*(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY+indY*(unsigned long long)geo.nVoxelX + indX; - voxelColumn[colIdx] = image[idx]; // Read the current volume value that we'll update by computing values from MULTIPLE projections (not just one) - // We'll be updating the local (register) variable, avoiding reads/writes from the slow main memory. - } // END copy 3D volume voxels to local array - - // Now iterate through projections -#pragma unroll - for(unsigned long projNumber=0; projNumber=totalNoOfProjections) - break; - - Point3D deltaX = projParamsArrayDev[6*projNumber]; // 6*projNumber because we have 6 Point3D values per projection - Point3D deltaY = projParamsArrayDev[6*projNumber+1]; - Point3D deltaZ = projParamsArrayDev[6*projNumber+2]; - Point3D xyzOrigin = projParamsArrayDev[6*projNumber+3]; - Point3D xyzOffset = projParamsArrayDev[6*projNumber+4]; - Point3D S = projParamsArrayDev[6*projNumber+5]; - - float sinalpha = projSinCosArrayDev[5*projNumber]; // 2*projNumber because we have 2 float (sin or cos angle) values per projection - float cosalpha = projSinCosArrayDev[5*projNumber+1]; - float COR = projSinCosArrayDev[5*projNumber+2]; - float DSD = projSinCosArrayDev[5*projNumber+3]; - float DSO = projSinCosArrayDev[5*projNumber+4]; - - float auxCOR=COR/geo.dDetecU; - // Now iterate through Z in our voxel column FOR A GIVEN PROJECTION -#pragma unroll - for(colIdx=0; colIdx=geo.nVoxelZ) - break; // break the loop. - - // "XYZ" in the scaled coordinate system of the current point. The image is rotated with the projection angles. - Point3D P; - P.x=(xyzOrigin.x+indX*deltaX.x+indY*deltaY.x+indZ*deltaZ.x); - P.y=(xyzOrigin.y+indX*deltaX.y+indY*deltaY.y+indZ*deltaZ.y)-auxCOR; - P.z=(xyzOrigin.z+indX*deltaX.z+indY*deltaY.z+indZ*deltaZ.z); - - // This is the vector defining the line from the source to the Voxel - float vectX,vectY,vectZ; - vectX=(P.x -S.x); - vectY=(P.y -S.y); - vectZ=(P.z -S.z); - - // Get the coordinates in the detector UV where the mid point of the voxel is projected. - float t=__fdividef(DSO-DSD-S.x,vectX); - float y,z; - y=vectY*t+S.y; - z=vectZ*t+S.z; - float u,v; - u=y+(float)geo.nDetecU*0.5f; - v=z+(float)geo.nDetecV*0.5f; - - float weight; - float realx,realy; - realx=-(geo.sVoxelX-geo.dVoxelX)*0.5f +indX*geo.dVoxelX +xyzOffset.x; - realy=-(geo.sVoxelY-geo.dVoxelY)*0.5f +indY*geo.dVoxelY +xyzOffset.y+COR; - - weight=__fdividef(DSO+realy*sinalpha-realx*cosalpha,DSO); - - weight=__frcp_rd(weight*weight); - - // Get Value in the computed (U,V) and multiply by the corresponding weight. - // indAlpha is the ABSOLUTE number of projection in the projection array (NOT the current number of projection set!) - -#if IS_FOR_MATLAB_TIGRE - voxelColumn[colIdx]+=tex3D(tex, v, u ,indAlpha+0.5f)*weight; -#else - voxelColumn[colIdx]+=tex3D(tex, u, v ,indAlpha+0.5f)*weight; -#endif - } // END iterating through column of voxels - - } // END iterating through multiple projections - - // And finally copy the updated local voxelColumn array back to our 3D volume (main memory) -#pragma unroll - for(colIdx=0; colIdx=geo.nVoxelZ) - break; // break the loop. - - unsigned long long idx =indZ*(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY+indY*(unsigned long long)geo.nVoxelX + indX; - image[idx] = voxelColumn[colIdx]; // Read the current volume value that we'll update by computing values from MULTIPLE projections (not just one) - // We'll be updating the local (register) variable, avoiding reads/writes from the slow main memory. - // According to references (Papenhausen), doing = is better than +=, since += requires main memory read followed by a write. - // We did all the reads into the local array at the BEGINNING of this kernel. According to Papenhausen, this type of read-write split is - // better for avoiding memory congestion. - } // END copy updated voxels from local array to our 3D volume - -} // END kernelPixelBackprojectionFDK - - - - -//______________________________________________________________________________ -// -// Function: voxel_backprojection -// -// Description: Main host function for FDK backprojection (invokes the kernel) -//______________________________________________________________________________ - -int voxel_backprojection(float * projections, Geometry geo, float* result,float const * const alphas, int nalpha, const GpuIds& gpuids) -{ - // printf("voxel_backprojection(geo.nDetector = %d, %d)\n", geo.nDetecU, geo.nDetecV); - // printf("geo.nVoxel = %d, %d, %d\n", geo.nVoxelX, geo.nVoxelY, geo.nVoxelZ); - - // Prepare for MultiGPU - int deviceCount = gpuids.GetLength(); - cudaCheckErrors("Device query fail"); - if (deviceCount == 0) { - mexErrMsgIdAndTxt("Atb:Voxel_backprojection:GPUselect","There are no available device(s) that support CUDA\n"); - } - - // CODE assumes - // 1.-All available devices are usable by this code - // 2.-All available devices are equal, they are the same machine (warning thrown) - // Check the available devices, and if they are the same - if (!gpuids.AreEqualDevices()) { - mexWarnMsgIdAndTxt("Atb:Voxel_backprojection:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); - } - - int dev; - // Split the CT problem - unsigned int split_image; - unsigned int split_projections; - splitCTbackprojection(gpuids,geo,nalpha,&split_image,&split_projections); - - - cudaCheckErrors("Error"); - //Pagelock memory for synchronous copy. - // Lets try to make the host memory pinned: - // We laredy queried the GPU and assuemd they are the same, thus should have the same attributes. - int isHostRegisterSupported = 0; -#if CUDART_VERSION >= 9020 - cudaDeviceGetAttribute(&isHostRegisterSupported,cudaDevAttrHostRegisterSupported,gpuids[0]); -#endif - // empirical testing shows that when the image split is smaller than 1 (also implies the image is not very big), the time to - // pin the memory is greater than the lost time in Synchronously launching the memcpys. This is only worth it when the image is too big. -#ifndef NO_PINNED_MEMORY - if (isHostRegisterSupported & (split_image>1 |deviceCount>1)){ - cudaHostRegister(result, (size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geo.nVoxelZ*(size_t)sizeof(float),cudaHostRegisterPortable); - } - if (isHostRegisterSupported ){ - cudaHostRegister(projections, (size_t)geo.nDetecU*(size_t)geo.nDetecV*(size_t)nalpha*(size_t)sizeof(float),cudaHostRegisterPortable); - } -#endif - cudaCheckErrors("Error pinning memory"); - - - // Create the arrays for the geometry. The main difference is that geo.offZ has been tuned for the - // image slices. The rest of the Geometry is the same - Geometry* geoArray=(Geometry*)malloc(split_image*deviceCount*sizeof(Geometry)); - createGeoArray(split_image*deviceCount,geo,geoArray,nalpha); - - // Now lest allocate all the image memory on the GPU, so we can use it later. If we have made our numbers correctly - // in the previous section this should leave enough space for the textures. - size_t num_bytes_img = (size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geoArray[0].nVoxelZ* sizeof(float); - float** dimage=(float**)malloc(deviceCount*sizeof(float*)); - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaMalloc((void**)&dimage[dev], num_bytes_img); - cudaCheckErrors("cudaMalloc fail"); - } - - //If it is the first time, lets make sure our image is zeroed. - int nStreamDevice=2; - int nStreams=deviceCount*nStreamDevice; - cudaStream_t* stream=(cudaStream_t*)malloc(nStreams*sizeof(cudaStream_t));; - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - for (int i = 0; i < nStreamDevice; ++i){ - cudaStreamCreate(&stream[i+dev*nStreamDevice]); - - } - } - - - - - // Kernel auxiliary variables - Point3D* projParamsArrayHost; - cudaMallocHost((void**)&projParamsArrayHost,6*PROJ_PER_KERNEL*sizeof(Point3D)); - float* projSinCosArrayHost; - cudaMallocHost((void**)&projSinCosArrayHost,5*PROJ_PER_KERNEL*sizeof(float)); - - - // Texture object variables - cudaTextureObject_t *texProj; - cudaArray **d_cuArrTex; - texProj =(cudaTextureObject_t*)malloc(deviceCount*2*sizeof(cudaTextureObject_t)); - d_cuArrTex =(cudaArray**)malloc(deviceCount*2*sizeof(cudaArray*)); - - // Auxiliary Host page-locked memory for fast and asycnornous memcpy. - - // Start with the main loop. The Projection data needs to be allocated and dealocated in the main loop - // as due to the nature of cudaArrays, we can not reuse them. This should not be a problem for the fast execution - // of the code, as repeated allocation and deallocation only happens when the projection data is very very big, - // and therefore allcoation time should be negligible, fluctuation of other computations should mask the time. - unsigned long long proj_linear_idx_start; - unsigned int proj_split_overlap_number; - unsigned int current_proj_split_size,current_proj_overlap_split_size; - size_t num_bytes_img_curr; - size_t img_linear_idx_start; - float** partial_projection; - size_t* proj_split_size; - - - - for(unsigned int img_slice=0;img_slice=proj_split_size[proj_block_split]) - break; // Exit the loop. Even when we leave the param arrays only partially filled, this is OK, since the kernel will check bounds anyway. - if(currProjNumber_global>=nalpha) - break; // Exit the loop. Even when we leave the param arrays only partially filled, this is OK, since the kernel will check bounds anyway. - - Point3D deltaX,deltaY,deltaZ,xyzOrigin, offOrig, /*offDetec,*/source; - float sinalpha,cosalpha; - - geoArray[img_slice*deviceCount+dev].alpha=-alphas[currProjNumber_global*3];//we got 3 angles now. - geoArray[img_slice*deviceCount+dev].theta=-alphas[currProjNumber_global*3+1]; - geoArray[img_slice*deviceCount+dev].psi =-alphas[currProjNumber_global*3+2]; - -// mexPrintf("%u %f \n",i,geoArray[img_slice*deviceCount+dev].alpha); -// mexPrintf("%u \n",currProjNumber_global); - - sinalpha=sin(geoArray[img_slice*deviceCount+dev].alpha); - cosalpha=cos(geoArray[img_slice*deviceCount+dev].alpha); - - projSinCosArrayHost[5*j]=sinalpha; // 2*j because we have 2 float (sin or cos angle) values per projection - projSinCosArrayHost[5*j+1]=cosalpha; - projSinCosArrayHost[5*j+2]=geo.COR[currProjNumber_global]; - projSinCosArrayHost[5*j+3]=geo.DSD[currProjNumber_global]; - projSinCosArrayHost[5*j+4]=geo.DSO[currProjNumber_global]; - - computeDeltasCube(geoArray[img_slice*deviceCount+dev],currProjNumber_global,&xyzOrigin,&deltaX,&deltaY,&deltaZ,&source); - - offOrig.x=geo.offOrigX[currProjNumber_global]; - offOrig.y=geo.offOrigY[currProjNumber_global]; - offOrig.z=geoArray[img_slice*deviceCount+dev].offOrigZ[currProjNumber_global]; - - projParamsArrayHost[6*j]=deltaX; // 6*j because we have 6 Point3D values per projection - projParamsArrayHost[6*j+1]=deltaY; - projParamsArrayHost[6*j+2]=deltaZ; - projParamsArrayHost[6*j+3]=xyzOrigin; - projParamsArrayHost[6*j+4]=offOrig; - projParamsArrayHost[6*j+5]=source; - } // END for (preparing params for kernel call) - - // Copy the prepared parameter arrays to constant memory to make it available for the kernel - cudaMemcpyToSymbolAsync(projSinCosArrayDev, projSinCosArrayHost, sizeof(float)*5*PROJ_PER_KERNEL,0,cudaMemcpyHostToDevice,stream[dev*nStreamDevice]); - cudaMemcpyToSymbolAsync(projParamsArrayDev, projParamsArrayHost, sizeof(Point3D)*6*PROJ_PER_KERNEL,0,cudaMemcpyHostToDevice,stream[dev*nStreamDevice]); - cudaStreamSynchronize(stream[dev*nStreamDevice]); - - kernelPixelBackprojectionFDK<<>>(geoArray[img_slice*deviceCount+dev],dimage[dev],i,proj_split_size[proj_block_split],texProj[(proj_block_split%2)*deviceCount+dev]); - } // END for - ////////////////////////////////////////////////////////////////////////////////////// - // END RB code, Main reconstruction loop: go through projections (rotation angles) and backproject - ////////////////////////////////////////////////////////////////////////////////////// - }// END for deviceCount - } // END sub-split of current projection chunk - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - } - - } // END projection splits - - - // Now we need to take the image out of the GPU - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - // We do not need to sycnronize because the array dealocators already do. - num_bytes_img_curr=(size_t)geoArray[img_slice*deviceCount+dev].nVoxelX*(size_t)geoArray[img_slice*deviceCount+dev].nVoxelY*(size_t)geoArray[img_slice*deviceCount+dev].nVoxelZ*sizeof(float); - img_linear_idx_start=(size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geoArray[0].nVoxelZ*(size_t)(img_slice*deviceCount+dev); - cudaMemcpyAsync(&result[img_linear_idx_start], dimage[dev], num_bytes_img_curr, cudaMemcpyDeviceToHost,stream[dev*nStreamDevice+1]); - } - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - cudaCheckErrors("Main loop fail"); - } - - } // end image splits - - ///////// Cleaning: - - - bool two_buffers_used=((((nalpha+split_projections-1)/split_projections)+PROJ_PER_KERNEL-1)/PROJ_PER_KERNEL)>1; - for(unsigned int i=0; i<2;i++){ // 2 buffers (if needed, maybe only 1) - if (!two_buffers_used && i==1) - break; - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDestroyTextureObject(texProj[i*deviceCount+dev]); - cudaFreeArray(d_cuArrTex[i*deviceCount+dev]); - } - } - cudaCheckErrors("cudadestroy textures result fail"); - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaFree(dimage[dev]); - } - cudaFreeHost(projSinCosArrayHost); - cudaFreeHost(projParamsArrayHost); - free(partial_projection); - free(proj_split_size); - - freeGeoArray(split_image*deviceCount,geoArray); -#ifndef NO_PINNED_MEMORY - if (isHostRegisterSupported & (split_image>1 |deviceCount>1)){ - cudaHostUnregister(result); - } - if (isHostRegisterSupported){ - cudaHostUnregister(projections); - } -#endif - - for (int i = 0; i < nStreams; ++i) - cudaStreamDestroy(stream[i]); - - cudaCheckErrors("cudaFree fail"); - - //cudaDeviceReset(); // For the Nvidia Visual Profiler - return 0; - -} // END voxel_backprojection -// - -void splitCTbackprojection(const GpuIds& gpuids, Geometry geo,int nalpha, unsigned int* split_image, unsigned int * split_projections){ - - - // We don't know if the devices are being used. lets check that. and only use the amount of memory we need. - - size_t mem_GPU_global; - checkFreeMemory(gpuids, &mem_GPU_global); - - const int deviceCount = gpuids.GetLength(); - - // Compute how much memory each of the relevant memory pieces need - size_t mem_image= (unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY*(unsigned long long)geo.nVoxelZ*sizeof(float); - size_t mem_proj= (unsigned long long)geo.nDetecU*(unsigned long long)geo.nDetecV*sizeof(float); - - - - - // Does everything fit in the GPU? - - if(mem_image/deviceCount+mem_proj*PROJ_PER_KERNEL*2(); - //cuda Array - cudaMalloc3DArray(&d_cuArrTex[dev], &channelDesc, extent); - - } - } - for (unsigned int dev = 0; dev < num_devices; dev++){ - cudaSetDevice(gpuids[dev]); - cudaMemcpy3DParms copyParams = {0}; - //Array creation - copyParams.srcPtr = make_cudaPitchedPtr((void *)projectiondata, extent.width*sizeof(float), extent.width, extent.height); - copyParams.dstArray = d_cuArrTex[dev]; - copyParams.extent = extent; - copyParams.kind = cudaMemcpyHostToDevice; - cudaMemcpy3DAsync(©Params,stream[dev*nStreamDevice+1]); - } - - //Array creation End - for (unsigned int dev = 0; dev < num_devices; dev++){ - cudaSetDevice(gpuids[dev]); - cudaResourceDesc texRes; - memset(&texRes, 0, sizeof(cudaResourceDesc)); - texRes.resType = cudaResourceTypeArray; - texRes.res.array.array = d_cuArrTex[dev]; - cudaTextureDesc texDescr; - memset(&texDescr, 0, sizeof(cudaTextureDesc)); - texDescr.normalizedCoords = false; - texDescr.filterMode = cudaFilterModeLinear; - texDescr.addressMode[0] = cudaAddressModeBorder; - texDescr.addressMode[1] = cudaAddressModeBorder; - texDescr.addressMode[2] = cudaAddressModeBorder; - texDescr.readMode = cudaReadModeElementType; - cudaCreateTextureObject(&texImage[dev], &texRes, &texDescr, NULL); - } -} - -//______________________________________________________________________________ -// -// Function: createGeoArray -// -// Description: This code generates the geometries needed to split the image properly in -// cases where the entire image does not fit in the memory of the GPU -//______________________________________________________________________________ - -void createGeoArray(unsigned int image_splits, Geometry geo,Geometry* geoArray, unsigned int nangles){ - - - unsigned int splitsize=(geo.nVoxelZ+image_splits-1)/image_splits; - - for(unsigned int sp=0;spx; - auxPoint.y=point->y; - auxPoint.z=point->z; - - // calculate sin and cos of 3 angles (used multiple times) - double sin_alpha, cos_alpha, sin_theta, cos_theta, sin_psi, cos_psi; - sin_alpha = sin((double)geo.alpha); - cos_alpha = cos((double)geo.alpha); - sin_theta = sin((double)geo.theta); - cos_theta = cos((double)geo.theta); - sin_psi = sin((double)geo.psi); - cos_psi = cos((double)geo.psi); - - point->x = auxPoint.x*(cos_psi*cos_theta*cos_alpha-sin_psi*sin_alpha) - +auxPoint.y*(-cos_psi*cos_theta*sin_alpha-sin_psi*cos_alpha) - +auxPoint.z*cos_psi*sin_theta; - point->y = auxPoint.x*(sin_psi*cos_theta*cos_alpha+cos_psi*sin_alpha) - +auxPoint.y*(-sin_psi*cos_theta*sin_alpha+cos_psi*cos_alpha) - +auxPoint.z*sin_psi*sin_theta; - point->z =-auxPoint.x*sin_theta*cos_alpha - +auxPoint.y*sin_theta*sin_alpha - +auxPoint.z*cos_theta; -} - -void rollPitchYawT(Geometry geo,int i, Point3Ddouble* point){ - - Point3Ddouble auxPoint; - auxPoint.x=point->x; - auxPoint.y=point->y; - auxPoint.z=point->z; - - // calculate sin and cos of 3 angles (used multiple times) - double sin_dRoll, cos_dRoll, sin_dPitch, cos_dPitch, sin_dYaw, cos_dYaw; - sin_dRoll = sin((double)geo.dRoll[i]); - cos_dRoll = cos((double)geo.dRoll[i]); - sin_dPitch = sin((double)geo.dPitch[i]); - cos_dPitch = cos((double)geo.dPitch[i]); - sin_dYaw = sin((double)geo.dYaw[i]); - cos_dYaw = cos((double)geo.dYaw[i]); - - point->x=cos_dRoll*cos_dPitch*auxPoint.x - +sin_dRoll*cos_dPitch*auxPoint.y - -sin_dPitch*auxPoint.z; - - point->y=(cos_dRoll*sin_dPitch*sin_dYaw - sin_dRoll*cos_dYaw)*auxPoint.x - +(sin_dRoll*sin_dPitch*sin_dYaw + cos_dRoll*cos_dYaw)*auxPoint.y - +cos_dPitch*sin_dYaw*auxPoint.z; - - point->z=(cos_dRoll*sin_dPitch*cos_dYaw + sin_dRoll*sin_dYaw)*auxPoint.x - +(sin_dRoll*sin_dPitch*cos_dYaw - cos_dRoll*sin_dYaw)*auxPoint.y - +cos_dPitch*cos_dYaw*auxPoint.z; -} - -//______________________________________________________________________________ -// -// Function: computeDeltasCube -// -// Description: Computes relative increments for each projection (volume rotation). -// Increments get passed to the backprojection kernel. -//______________________________________________________________________________ - -void computeDeltasCube(Geometry geo,int i, Point3D* xyzorigin, Point3D* deltaX, Point3D* deltaY, Point3D* deltaZ,Point3D* S) -{ - - // initialize points with double precision - Point3Ddouble P, Px,Py,Pz; - - // Get coords of Img(0,0,0) - P.x=-(geo.sVoxelX/2-geo.dVoxelX/2)+geo.offOrigX[i]; - P.y=-(geo.sVoxelY/2-geo.dVoxelY/2)+geo.offOrigY[i]; - P.z=-(geo.sVoxelZ/2-geo.dVoxelZ/2)+geo.offOrigZ[i]; - - // Get coords from next voxel in each direction - Px.x=P.x+geo.dVoxelX; Py.x=P.x; Pz.x=P.x; - Px.y=P.y; Py.y=P.y+geo.dVoxelY; Pz.y=P.y; - Px.z=P.z; Py.z=P.z; Pz.z=P.z+geo.dVoxelZ; - - // Rotate image around X axis (this is equivalent of rotating the source and detector) RZ RY RZ - eulerZYZT(geo,&P); - eulerZYZT(geo,&Px); - eulerZYZT(geo,&Py); - eulerZYZT(geo,&Pz); - - //detector offset - P.z =P.z-geo.offDetecV[i]; P.y =P.y-geo.offDetecU[i]; - Px.z =Px.z-geo.offDetecV[i]; Px.y =Px.y-geo.offDetecU[i]; - Py.z =Py.z-geo.offDetecV[i]; Py.y =Py.y-geo.offDetecU[i]; - Pz.z =Pz.z-geo.offDetecV[i]; Pz.y =Pz.y-geo.offDetecU[i]; - - //Detector Roll pitch Yaw - // - // first, we need to offset everything so (0,0,0) is the center of the detector - // Only X is required for that - P.x=P.x+(geo.DSD[i]-geo.DSO[i]); - Px.x=Px.x+(geo.DSD[i]-geo.DSO[i]); - Py.x=Py.x+(geo.DSD[i]-geo.DSO[i]); - Pz.x=Pz.x+(geo.DSD[i]-geo.DSO[i]); - rollPitchYawT(geo,i,&P); - rollPitchYawT(geo,i,&Px); - rollPitchYawT(geo,i,&Py); - rollPitchYawT(geo,i,&Pz); - - P.x=P.x-(geo.DSD[i]-geo.DSO[i]); - Px.x=Px.x-(geo.DSD[i]-geo.DSO[i]); - Py.x=Py.x-(geo.DSD[i]-geo.DSO[i]); - Pz.x=Pz.x-(geo.DSD[i]-geo.DSO[i]); - //Done for P, now source - Point3Ddouble source; - source.x=geo.DSD[i]; //already offseted for rotation - source.y=-geo.offDetecU[i]; - source.z=-geo.offDetecV[i]; - rollPitchYawT(geo,i,&source); - - source.x=source.x-(geo.DSD[i]-geo.DSO[i]);// source.y=source.y-auxOff.y; source.z=source.z-auxOff.z; - -// mexPrintf("%f,%f,%f\n",source.x,source.y,source.z); - // Scale coords so detector pixels are 1x1 - - P.z =P.z /geo.dDetecV; P.y =P.y/geo.dDetecU; - Px.z=Px.z/geo.dDetecV; Px.y=Px.y/geo.dDetecU; - Py.z=Py.z/geo.dDetecV; Py.y=Py.y/geo.dDetecU; - Pz.z=Pz.z/geo.dDetecV; Pz.y=Pz.y/geo.dDetecU; - - source.z=source.z/geo.dDetecV; source.y=source.y/geo.dDetecU; - - // get deltas of the changes in voxels - deltaX->x=Px.x-P.x; deltaX->y=Px.y-P.y; deltaX->z=Px.z-P.z; - deltaY->x=Py.x-P.x; deltaY->y=Py.y-P.y; deltaY->z=Py.z-P.z; - deltaZ->x=Pz.x-P.x; deltaZ->y=Pz.y-P.y; deltaZ->z=Pz.z-P.z; - - // cast the results from the double precision calculations back to float - *xyzorigin=P.to_float(); - *S=source.to_float(); -} - -void checkFreeMemory(const GpuIds& gpuids,size_t *mem_GPU_global){ - size_t memfree; - size_t memtotal; - const int deviceCount = gpuids.GetLength(); - - for (int dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaMemGetInfo(&memfree,&memtotal); - if(dev==0) *mem_GPU_global=memfree; - if(memfree +#include +#include +#include "voxel_backprojection.hpp" +#include "TIGRE_common.hpp" +#include +#include "GpuIds.hpp" + +// https://stackoverflow.com/questions/16282136/is-there-a-cuda-equivalent-of-perror +#define hipCheckErrors(msg) \ +do { \ + hipError_t __err = hipGetLastError(); \ + if (__err != hipSuccess) { \ + mexPrintf("%s \n",msg);\ + mexErrMsgIdAndTxt("CBCT:CUDA:Atb",hipGetErrorString(__err));\ + } \ +} while (0) + + +#define MAXTREADS 1024 + /*GEOMETRY DEFINITION + * + * Detector plane, behind + * |-----------------------------| + * | | + * | | + * | | + * | | + * | +--------+ | + * | / /| | + * A Z | / / |*D | + * | | +--------+ | | + * | | | | | | + * | | | *O | + | + * *--->y | | | / | + * / | | |/ | + * V X | +--------+ | + * |-----------------------------| + * + * *S + * + * + * + * + * + **/ + + void CreateTexture(const GpuIds& gpuids,float* projectiondata,Geometry geo,hipArray** d_cuArrTex,unsigned int nangles, hipTextureObject_t *texImage,hipStream_t* stream, int nStreamDevice,bool allocate); + + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// RB, 10/31/2016: Add constant memory arrays to store parameters for all projections to be analyzed during a single kernel call +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// The optimal values of two constants obtained by RB on NVIDIA Quadro K2200 (4 GB RAM, 640 CUDA cores) for 512^3 volume and 512^3 projections (512 proj, each 512 x 512) were: +// PROJ_PER_KERNEL = 32 or 16 (very similar times) +// VOXELS_PER_THREAD = 8 +// Speedup of the entire FDK backprojection (not only kernel run, also memcpy etc.) was nearly 4x relative to the original (single projection, single voxel per thread) code. +// (e.g. 16.2 s vs. ~62 s). + +const int PROJ_PER_KERNEL = 32; // Number of 2D projections to be analyzed by a single thread. This can be tweaked to see what works best. 32 was the optimal value in the paper by Zinsser and Keck. +const int VOXELS_PER_THREAD = 8; // Number of voxels to be computed by s single thread. Can be tweaked to see what works best. 4 was the optimal value in the paper by Zinsser and Keck. + +// We have PROJ_PER_KERNEL projections and we need 6 parameters for each projection: +// deltaX, deltaY, deltaZ, xyzOrigin, offOrig, offDetec +// So we need to keep PROJ_PER_KERNEL*6 values in our deltas array FOR EACH CALL to our main kernel +// (they will be updated in the main loop before each kernel call). + +__constant__ Point3D projParamsArrayDev[6*PROJ_PER_KERNEL]; // Dev means it is on device + +// We also need a corresponding array on the host side to be filled before each kernel call, then copied to the device (array in constant memory above) +// Point3D projParamsArrayHost[6*PROJ_PER_KERNEL]; // Host means it is host memory + +// Now we also need to store sinAlpha and cosAlpha for each projection (two floats per projection) +__constant__ float projSinCosArrayDev[5*PROJ_PER_KERNEL]; + + + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// END RB, 10/31/2016: Add constant memory arrays to store parameters for all projections to be analyzed during a single kernel call +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + +//______________________________________________________________________________ +// +// Function: kernelPixelBackprojectionFDK +// +// Description: Main FDK backprojection kernel +//______________________________________________________________________________ + +__global__ void kernelPixelBackprojectionFDK(const Geometry geo, float* image,const int currProjSetNumber, const int totalNoOfProjections, hipTextureObject_t tex) +{ + + // Old kernel call signature: + // kernelPixelBackprojectionFDK<<>>(geo,dimage,i,deltaX,deltaY,deltaZ,xyzOrigin,offOrig,offDetec,sinalpha,cosalpha); + // We just read in most of the params from the constant memory instead of getting them from the param list. + // This is because we now have MANY params, since single kernel processes more than one projection! + /* __global__ void kernelPixelBackprojectionFDK(const Geometry geo, + * float* image, + * const int indAlpha, + * const Point3D deltaX , + * const Point3D deltaY, + * const Point3D deltaZ, + * const Point3D xyzOrigin, + * const Point3D xyzOffset, + * const Point3D uv0Offset, + * const float sinalpha, + * const float cosalpha){ + */ + unsigned long long indY = blockIdx.y * blockDim.y + threadIdx.y; + unsigned long long indX = blockIdx.x * blockDim.x + threadIdx.x; + // unsigned long startIndZ = blockIdx.z * blockDim.z + threadIdx.z; // This is only STARTING z index of the column of voxels that the thread will handle + unsigned long long startIndZ = blockIdx.z * VOXELS_PER_THREAD + threadIdx.z; // This is only STARTING z index of the column of voxels that the thread will handle + //Make sure we don't go out of bounds + if (indX>=geo.nVoxelX || indY>=geo.nVoxelY || startIndZ>=geo.nVoxelZ) + return; + + // We'll keep a local auxiliary array of values of a column of voxels that this thread will update + float voxelColumn[VOXELS_PER_THREAD]; + + // First we need to copy the curent 3D volume values from the column to our auxiliary array so that we can then + // work on them (update them by computing values from multiple projections) locally - avoiding main memory reads/writes + + unsigned long colIdx; +#pragma unroll + for(colIdx=0; colIdx=geo.nVoxelZ) + break; // break the loop. + + unsigned long long idx =indZ*(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY+indY*(unsigned long long)geo.nVoxelX + indX; + voxelColumn[colIdx] = image[idx]; // Read the current volume value that we'll update by computing values from MULTIPLE projections (not just one) + // We'll be updating the local (register) variable, avoiding reads/writes from the slow main memory. + } // END copy 3D volume voxels to local array + + // Now iterate through projections +#pragma unroll + for(unsigned long projNumber=0; projNumber=totalNoOfProjections) + break; + + Point3D deltaX = projParamsArrayDev[6*projNumber]; // 6*projNumber because we have 6 Point3D values per projection + Point3D deltaY = projParamsArrayDev[6*projNumber+1]; + Point3D deltaZ = projParamsArrayDev[6*projNumber+2]; + Point3D xyzOrigin = projParamsArrayDev[6*projNumber+3]; + Point3D xyzOffset = projParamsArrayDev[6*projNumber+4]; + Point3D S = projParamsArrayDev[6*projNumber+5]; + + float sinalpha = projSinCosArrayDev[5*projNumber]; // 2*projNumber because we have 2 float (sin or cos angle) values per projection + float cosalpha = projSinCosArrayDev[5*projNumber+1]; + float COR = projSinCosArrayDev[5*projNumber+2]; + float DSD = projSinCosArrayDev[5*projNumber+3]; + float DSO = projSinCosArrayDev[5*projNumber+4]; + + float auxCOR=COR/geo.dDetecU; + // Now iterate through Z in our voxel column FOR A GIVEN PROJECTION +#pragma unroll + for(colIdx=0; colIdx=geo.nVoxelZ) + break; // break the loop. + + // "XYZ" in the scaled coordinate system of the current point. The image is rotated with the projection angles. + Point3D P; + P.x=(xyzOrigin.x+indX*deltaX.x+indY*deltaY.x+indZ*deltaZ.x); + P.y=(xyzOrigin.y+indX*deltaX.y+indY*deltaY.y+indZ*deltaZ.y)-auxCOR; + P.z=(xyzOrigin.z+indX*deltaX.z+indY*deltaY.z+indZ*deltaZ.z); + + // This is the vector defining the line from the source to the Voxel + float vectX,vectY,vectZ; + vectX=(P.x -S.x); + vectY=(P.y -S.y); + vectZ=(P.z -S.z); + + // Get the coordinates in the detector UV where the mid point of the voxel is projected. + float t=__fdividef(DSO-DSD-S.x,vectX); + float y,z; + y=vectY*t+S.y; + z=vectZ*t+S.z; + float u,v; + u=y+(float)geo.nDetecU*0.5f; + v=z+(float)geo.nDetecV*0.5f; + + float weight; + float realx,realy; + realx=-(geo.sVoxelX-geo.dVoxelX)*0.5f +indX*geo.dVoxelX +xyzOffset.x; + realy=-(geo.sVoxelY-geo.dVoxelY)*0.5f +indY*geo.dVoxelY +xyzOffset.y+COR; + + weight=__fdividef(DSO+realy*sinalpha-realx*cosalpha,DSO); + + weight=__frcp_rd(weight*weight); + + // Get Value in the computed (U,V) and multiply by the corresponding weight. + // indAlpha is the ABSOLUTE number of projection in the projection array (NOT the current number of projection set!) + +#if IS_FOR_MATLAB_TIGRE + voxelColumn[colIdx]+=tex3D(tex, v, u ,indAlpha+0.5f)*weight; +#else + voxelColumn[colIdx]+=tex3D(tex, u, v ,indAlpha+0.5f)*weight; +#endif + } // END iterating through column of voxels + + } // END iterating through multiple projections + + // And finally copy the updated local voxelColumn array back to our 3D volume (main memory) +#pragma unroll + for(colIdx=0; colIdx=geo.nVoxelZ) + break; // break the loop. + + unsigned long long idx =indZ*(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY+indY*(unsigned long long)geo.nVoxelX + indX; + image[idx] = voxelColumn[colIdx]; // Read the current volume value that we'll update by computing values from MULTIPLE projections (not just one) + // We'll be updating the local (register) variable, avoiding reads/writes from the slow main memory. + // According to references (Papenhausen), doing = is better than +=, since += requires main memory read followed by a write. + // We did all the reads into the local array at the BEGINNING of this kernel. According to Papenhausen, this type of read-write split is + // better for avoiding memory congestion. + } // END copy updated voxels from local array to our 3D volume + +} // END kernelPixelBackprojectionFDK + + + + +//______________________________________________________________________________ +// +// Function: voxel_backprojection +// +// Description: Main host function for FDK backprojection (invokes the kernel) +//______________________________________________________________________________ + +int voxel_backprojection(float * projections, Geometry geo, float* result,float const * const alphas, int nalpha, const GpuIds& gpuids) +{ + // printf("voxel_backprojection(geo.nDetector = %d, %d)\n", geo.nDetecU, geo.nDetecV); + // printf("geo.nVoxel = %d, %d, %d\n", geo.nVoxelX, geo.nVoxelY, geo.nVoxelZ); + + // Prepare for MultiGPU + int deviceCount = gpuids.GetLength(); + hipCheckErrors("Device query fail"); + if (deviceCount == 0) { + mexErrMsgIdAndTxt("Atb:Voxel_backprojection:GPUselect","There are no available device(s) that support CUDA\n"); + } + + // CODE assumes + // 1.-All available devices are usable by this code + // 2.-All available devices are equal, they are the same machine (warning thrown) + // Check the available devices, and if they are the same + if (!gpuids.AreEqualDevices()) { + mexWarnMsgIdAndTxt("Atb:Voxel_backprojection:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); + } + + int dev; + // Split the CT problem + unsigned int split_image; + unsigned int split_projections; + splitCTbackprojection(gpuids,geo,nalpha,&split_image,&split_projections); + + + hipCheckErrors("Error"); + //Pagelock memory for synchronous copy. + // Lets try to make the host memory pinned: + // We laredy queried the GPU and assuemd they are the same, thus should have the same attributes. + int isHostRegisterSupported = 0; +#if CUDART_VERSION >= 9020 + hipDeviceGetAttribute(&isHostRegisterSupported,hipDevAttrHostRegisterSupported,gpuids[0]); +#endif + // empirical testing shows that when the image split is smaller than 1 (also implies the image is not very big), the time to + // pin the memory is greater than the lost time in Synchronously launching the memcpys. This is only worth it when the image is too big. +#ifndef NO_PINNED_MEMORY + if (isHostRegisterSupported & (split_image>1 |deviceCount>1)){ + hipHostRegister(result, (size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geo.nVoxelZ*(size_t)sizeof(float),hipHostRegisterPortable); + } + if (isHostRegisterSupported ){ + hipHostRegister(projections, (size_t)geo.nDetecU*(size_t)geo.nDetecV*(size_t)nalpha*(size_t)sizeof(float),hipHostRegisterPortable); + } +#endif + hipCheckErrors("Error pinning memory"); + + + // Create the arrays for the geometry. The main difference is that geo.offZ has been tuned for the + // image slices. The rest of the Geometry is the same + Geometry* geoArray=(Geometry*)malloc(split_image*deviceCount*sizeof(Geometry)); + createGeoArray(split_image*deviceCount,geo,geoArray,nalpha); + + // Now lest allocate all the image memory on the GPU, so we can use it later. If we have made our numbers correctly + // in the previous section this should leave enough space for the textures. + size_t num_bytes_img = (size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geoArray[0].nVoxelZ* sizeof(float); + float** dimage=(float**)malloc(deviceCount*sizeof(float*)); + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipMalloc((void**)&dimage[dev], num_bytes_img); + hipCheckErrors("hipMalloc fail"); + } + + //If it is the first time, lets make sure our image is zeroed. + int nStreamDevice=2; + int nStreams=deviceCount*nStreamDevice; + hipStream_t* stream=(hipStream_t*)malloc(nStreams*sizeof(hipStream_t));; + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + for (int i = 0; i < nStreamDevice; ++i){ + hipStreamCreate(&stream[i+dev*nStreamDevice]); + + } + } + + + + + // Kernel auxiliary variables + Point3D* projParamsArrayHost; + hipMallocHost((void**)&projParamsArrayHost,6*PROJ_PER_KERNEL*sizeof(Point3D)); + float* projSinCosArrayHost; + hipMallocHost((void**)&projSinCosArrayHost,5*PROJ_PER_KERNEL*sizeof(float)); + + + // Texture object variables + hipTextureObject_t *texProj; + hipArray **d_cuArrTex; + texProj =(hipTextureObject_t*)malloc(deviceCount*2*sizeof(hipTextureObject_t)); + d_cuArrTex =(hipArray**)malloc(deviceCount*2*sizeof(hipArray*)); + + // Auxiliary Host page-locked memory for fast and asycnornous memcpy. + + // Start with the main loop. The Projection data needs to be allocated and dealocated in the main loop + // as due to the nature of hipArrays, we can not reuse them. This should not be a problem for the fast execution + // of the code, as repeated allocation and deallocation only happens when the projection data is very very big, + // and therefore allcoation time should be negligible, fluctuation of other computations should mask the time. + unsigned long long proj_linear_idx_start; + unsigned int proj_split_overlap_number; + unsigned int current_proj_split_size,current_proj_overlap_split_size; + size_t num_bytes_img_curr; + size_t img_linear_idx_start; + float** partial_projection; + size_t* proj_split_size; + + + + for(unsigned int img_slice=0;img_slice=proj_split_size[proj_block_split]) + break; // Exit the loop. Even when we leave the param arrays only partially filled, this is OK, since the kernel will check bounds anyway. + if(currProjNumber_global>=nalpha) + break; // Exit the loop. Even when we leave the param arrays only partially filled, this is OK, since the kernel will check bounds anyway. + + Point3D deltaX,deltaY,deltaZ,xyzOrigin, offOrig, /*offDetec,*/source; + float sinalpha,cosalpha; + + geoArray[img_slice*deviceCount+dev].alpha=-alphas[currProjNumber_global*3];//we got 3 angles now. + geoArray[img_slice*deviceCount+dev].theta=-alphas[currProjNumber_global*3+1]; + geoArray[img_slice*deviceCount+dev].psi =-alphas[currProjNumber_global*3+2]; + +// mexPrintf("%u %f \n",i,geoArray[img_slice*deviceCount+dev].alpha); +// mexPrintf("%u \n",currProjNumber_global); + + sinalpha=sin(geoArray[img_slice*deviceCount+dev].alpha); + cosalpha=cos(geoArray[img_slice*deviceCount+dev].alpha); + + projSinCosArrayHost[5*j]=sinalpha; // 2*j because we have 2 float (sin or cos angle) values per projection + projSinCosArrayHost[5*j+1]=cosalpha; + projSinCosArrayHost[5*j+2]=geo.COR[currProjNumber_global]; + projSinCosArrayHost[5*j+3]=geo.DSD[currProjNumber_global]; + projSinCosArrayHost[5*j+4]=geo.DSO[currProjNumber_global]; + + computeDeltasCube(geoArray[img_slice*deviceCount+dev],currProjNumber_global,&xyzOrigin,&deltaX,&deltaY,&deltaZ,&source); + + offOrig.x=geo.offOrigX[currProjNumber_global]; + offOrig.y=geo.offOrigY[currProjNumber_global]; + offOrig.z=geoArray[img_slice*deviceCount+dev].offOrigZ[currProjNumber_global]; + + projParamsArrayHost[6*j]=deltaX; // 6*j because we have 6 Point3D values per projection + projParamsArrayHost[6*j+1]=deltaY; + projParamsArrayHost[6*j+2]=deltaZ; + projParamsArrayHost[6*j+3]=xyzOrigin; + projParamsArrayHost[6*j+4]=offOrig; + projParamsArrayHost[6*j+5]=source; + } // END for (preparing params for kernel call) + + // Copy the prepared parameter arrays to constant memory to make it available for the kernel + hipMemcpyToSymbolAsync(projSinCosArrayDev, projSinCosArrayHost, sizeof(float)*5*PROJ_PER_KERNEL,0,hipMemcpyHostToDevice,stream[dev*nStreamDevice]); + hipMemcpyToSymbolAsync(projParamsArrayDev, projParamsArrayHost, sizeof(Point3D)*6*PROJ_PER_KERNEL,0,hipMemcpyHostToDevice,stream[dev*nStreamDevice]); + hipStreamSynchronize(stream[dev*nStreamDevice]); + + kernelPixelBackprojectionFDK<<>>(geoArray[img_slice*deviceCount+dev],dimage[dev],i,proj_split_size[proj_block_split],texProj[(proj_block_split%2)*deviceCount+dev]); + } // END for + ////////////////////////////////////////////////////////////////////////////////////// + // END RB code, Main reconstruction loop: go through projections (rotation angles) and backproject + ////////////////////////////////////////////////////////////////////////////////////// + }// END for deviceCount + } // END sub-split of current projection chunk + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + } + + } // END projection splits + + + // Now we need to take the image out of the GPU + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + // We do not need to sycnronize because the array dealocators already do. + num_bytes_img_curr=(size_t)geoArray[img_slice*deviceCount+dev].nVoxelX*(size_t)geoArray[img_slice*deviceCount+dev].nVoxelY*(size_t)geoArray[img_slice*deviceCount+dev].nVoxelZ*sizeof(float); + img_linear_idx_start=(size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geoArray[0].nVoxelZ*(size_t)(img_slice*deviceCount+dev); + hipMemcpyAsync(&result[img_linear_idx_start], dimage[dev], num_bytes_img_curr, hipMemcpyDeviceToHost,stream[dev*nStreamDevice+1]); + } + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + hipCheckErrors("Main loop fail"); + } + + } // end image splits + + ///////// Cleaning: + + + bool two_buffers_used=((((nalpha+split_projections-1)/split_projections)+PROJ_PER_KERNEL-1)/PROJ_PER_KERNEL)>1; + for(unsigned int i=0; i<2;i++){ // 2 buffers (if needed, maybe only 1) + if (!two_buffers_used && i==1) + break; + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDestroyTextureObject(texProj[i*deviceCount+dev]); + hipFreeArray(d_cuArrTex[i*deviceCount+dev]); + } + } + hipCheckErrors("cudadestroy textures result fail"); + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipFree(dimage[dev]); + } + hipFreeHost(projSinCosArrayHost); + hipFreeHost(projParamsArrayHost); + free(partial_projection); + free(proj_split_size); + + freeGeoArray(split_image*deviceCount,geoArray); +#ifndef NO_PINNED_MEMORY + if (isHostRegisterSupported & (split_image>1 |deviceCount>1)){ + hipHostUnregister(result); + } + if (isHostRegisterSupported){ + hipHostUnregister(projections); + } +#endif + + for (int i = 0; i < nStreams; ++i) + hipStreamDestroy(stream[i]); + + hipCheckErrors("hipFree fail"); + + //hipDeviceReset(); // For the Nvidia Visual Profiler + return 0; + +} // END voxel_backprojection +// + +void splitCTbackprojection(const GpuIds& gpuids, Geometry geo,int nalpha, unsigned int* split_image, unsigned int * split_projections){ + + + // We don't know if the devices are being used. lets check that. and only use the amount of memory we need. + + size_t mem_GPU_global; + checkFreeMemory(gpuids, &mem_GPU_global); + + const int deviceCount = gpuids.GetLength(); + + // Compute how much memory each of the relevant memory pieces need + size_t mem_image= (unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY*(unsigned long long)geo.nVoxelZ*sizeof(float); + size_t mem_proj= (unsigned long long)geo.nDetecU*(unsigned long long)geo.nDetecV*sizeof(float); + + + + + // Does everything fit in the GPU? + + if(mem_image/deviceCount+mem_proj*PROJ_PER_KERNEL*2(); + //cuda Array + hipMalloc3DArray(&d_cuArrTex[dev], &channelDesc, extent); + + } + } + for (unsigned int dev = 0; dev < num_devices; dev++){ + hipSetDevice(gpuids[dev]); + hipMemcpy3DParms copyParams = {0}; + //Array creation + copyParams.srcPtr = make_cudaPitchedPtr((void *)projectiondata, extent.width*sizeof(float), extent.width, extent.height); + copyParams.dstArray = d_cuArrTex[dev]; + copyParams.extent = extent; + copyParams.kind = hipMemcpyHostToDevice; + hipMemcpy3DAsync(©Params,stream[dev*nStreamDevice+1]); + } + + //Array creation End + for (unsigned int dev = 0; dev < num_devices; dev++){ + hipSetDevice(gpuids[dev]); + hipResourceDesc texRes; + memset(&texRes, 0, sizeof(hipResourceDesc)); + texRes.resType = hipResourceTypeArray; + texRes.res.array.array = d_cuArrTex[dev]; + hipTextureDesc texDescr; + memset(&texDescr, 0, sizeof(hipTextureDesc)); + texDescr.normalizedCoords = false; + texDescr.filterMode = hipFilterModeLinear; + texDescr.addressMode[0] = hipAddressModeBorder; + texDescr.addressMode[1] = hipAddressModeBorder; + texDescr.addressMode[2] = hipAddressModeBorder; + texDescr.readMode = hipReadModeElementType; + hipCreateTextureObject(&texImage[dev], &texRes, &texDescr, NULL); + } +} + +//______________________________________________________________________________ +// +// Function: createGeoArray +// +// Description: This code generates the geometries needed to split the image properly in +// cases where the entire image does not fit in the memory of the GPU +//______________________________________________________________________________ + +void createGeoArray(unsigned int image_splits, Geometry geo,Geometry* geoArray, unsigned int nangles){ + + + unsigned int splitsize=(geo.nVoxelZ+image_splits-1)/image_splits; + + for(unsigned int sp=0;spx; + auxPoint.y=point->y; + auxPoint.z=point->z; + + // calculate sin and cos of 3 angles (used multiple times) + double sin_alpha, cos_alpha, sin_theta, cos_theta, sin_psi, cos_psi; + sin_alpha = sin((double)geo.alpha); + cos_alpha = cos((double)geo.alpha); + sin_theta = sin((double)geo.theta); + cos_theta = cos((double)geo.theta); + sin_psi = sin((double)geo.psi); + cos_psi = cos((double)geo.psi); + + point->x = auxPoint.x*(cos_psi*cos_theta*cos_alpha-sin_psi*sin_alpha) + +auxPoint.y*(-cos_psi*cos_theta*sin_alpha-sin_psi*cos_alpha) + +auxPoint.z*cos_psi*sin_theta; + point->y = auxPoint.x*(sin_psi*cos_theta*cos_alpha+cos_psi*sin_alpha) + +auxPoint.y*(-sin_psi*cos_theta*sin_alpha+cos_psi*cos_alpha) + +auxPoint.z*sin_psi*sin_theta; + point->z =-auxPoint.x*sin_theta*cos_alpha + +auxPoint.y*sin_theta*sin_alpha + +auxPoint.z*cos_theta; +} + +void rollPitchYawT(Geometry geo,int i, Point3Ddouble* point){ + + Point3Ddouble auxPoint; + auxPoint.x=point->x; + auxPoint.y=point->y; + auxPoint.z=point->z; + + // calculate sin and cos of 3 angles (used multiple times) + double sin_dRoll, cos_dRoll, sin_dPitch, cos_dPitch, sin_dYaw, cos_dYaw; + sin_dRoll = sin((double)geo.dRoll[i]); + cos_dRoll = cos((double)geo.dRoll[i]); + sin_dPitch = sin((double)geo.dPitch[i]); + cos_dPitch = cos((double)geo.dPitch[i]); + sin_dYaw = sin((double)geo.dYaw[i]); + cos_dYaw = cos((double)geo.dYaw[i]); + + point->x=cos_dRoll*cos_dPitch*auxPoint.x + +sin_dRoll*cos_dPitch*auxPoint.y + -sin_dPitch*auxPoint.z; + + point->y=(cos_dRoll*sin_dPitch*sin_dYaw - sin_dRoll*cos_dYaw)*auxPoint.x + +(sin_dRoll*sin_dPitch*sin_dYaw + cos_dRoll*cos_dYaw)*auxPoint.y + +cos_dPitch*sin_dYaw*auxPoint.z; + + point->z=(cos_dRoll*sin_dPitch*cos_dYaw + sin_dRoll*sin_dYaw)*auxPoint.x + +(sin_dRoll*sin_dPitch*cos_dYaw - cos_dRoll*sin_dYaw)*auxPoint.y + +cos_dPitch*cos_dYaw*auxPoint.z; +} + +//______________________________________________________________________________ +// +// Function: computeDeltasCube +// +// Description: Computes relative increments for each projection (volume rotation). +// Increments get passed to the backprojection kernel. +//______________________________________________________________________________ + +void computeDeltasCube(Geometry geo,int i, Point3D* xyzorigin, Point3D* deltaX, Point3D* deltaY, Point3D* deltaZ,Point3D* S) +{ + + // initialize points with double precision + Point3Ddouble P, Px,Py,Pz; + + // Get coords of Img(0,0,0) + P.x=-(geo.sVoxelX/2-geo.dVoxelX/2)+geo.offOrigX[i]; + P.y=-(geo.sVoxelY/2-geo.dVoxelY/2)+geo.offOrigY[i]; + P.z=-(geo.sVoxelZ/2-geo.dVoxelZ/2)+geo.offOrigZ[i]; + + // Get coords from next voxel in each direction + Px.x=P.x+geo.dVoxelX; Py.x=P.x; Pz.x=P.x; + Px.y=P.y; Py.y=P.y+geo.dVoxelY; Pz.y=P.y; + Px.z=P.z; Py.z=P.z; Pz.z=P.z+geo.dVoxelZ; + + // Rotate image around X axis (this is equivalent of rotating the source and detector) RZ RY RZ + eulerZYZT(geo,&P); + eulerZYZT(geo,&Px); + eulerZYZT(geo,&Py); + eulerZYZT(geo,&Pz); + + //detector offset + P.z =P.z-geo.offDetecV[i]; P.y =P.y-geo.offDetecU[i]; + Px.z =Px.z-geo.offDetecV[i]; Px.y =Px.y-geo.offDetecU[i]; + Py.z =Py.z-geo.offDetecV[i]; Py.y =Py.y-geo.offDetecU[i]; + Pz.z =Pz.z-geo.offDetecV[i]; Pz.y =Pz.y-geo.offDetecU[i]; + + //Detector Roll pitch Yaw + // + // first, we need to offset everything so (0,0,0) is the center of the detector + // Only X is required for that + P.x=P.x+(geo.DSD[i]-geo.DSO[i]); + Px.x=Px.x+(geo.DSD[i]-geo.DSO[i]); + Py.x=Py.x+(geo.DSD[i]-geo.DSO[i]); + Pz.x=Pz.x+(geo.DSD[i]-geo.DSO[i]); + rollPitchYawT(geo,i,&P); + rollPitchYawT(geo,i,&Px); + rollPitchYawT(geo,i,&Py); + rollPitchYawT(geo,i,&Pz); + + P.x=P.x-(geo.DSD[i]-geo.DSO[i]); + Px.x=Px.x-(geo.DSD[i]-geo.DSO[i]); + Py.x=Py.x-(geo.DSD[i]-geo.DSO[i]); + Pz.x=Pz.x-(geo.DSD[i]-geo.DSO[i]); + //Done for P, now source + Point3Ddouble source; + source.x=geo.DSD[i]; //already offseted for rotation + source.y=-geo.offDetecU[i]; + source.z=-geo.offDetecV[i]; + rollPitchYawT(geo,i,&source); + + source.x=source.x-(geo.DSD[i]-geo.DSO[i]);// source.y=source.y-auxOff.y; source.z=source.z-auxOff.z; + +// mexPrintf("%f,%f,%f\n",source.x,source.y,source.z); + // Scale coords so detector pixels are 1x1 + + P.z =P.z /geo.dDetecV; P.y =P.y/geo.dDetecU; + Px.z=Px.z/geo.dDetecV; Px.y=Px.y/geo.dDetecU; + Py.z=Py.z/geo.dDetecV; Py.y=Py.y/geo.dDetecU; + Pz.z=Pz.z/geo.dDetecV; Pz.y=Pz.y/geo.dDetecU; + + source.z=source.z/geo.dDetecV; source.y=source.y/geo.dDetecU; + + // get deltas of the changes in voxels + deltaX->x=Px.x-P.x; deltaX->y=Px.y-P.y; deltaX->z=Px.z-P.z; + deltaY->x=Py.x-P.x; deltaY->y=Py.y-P.y; deltaY->z=Py.z-P.z; + deltaZ->x=Pz.x-P.x; deltaZ->y=Pz.y-P.y; deltaZ->z=Pz.z-P.z; + + // cast the results from the double precision calculations back to float + *xyzorigin=P.to_float(); + *S=source.to_float(); +} + +void checkFreeMemory(const GpuIds& gpuids,size_t *mem_GPU_global){ + size_t memfree; + size_t memtotal; + const int deviceCount = gpuids.GetLength(); + + for (int dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipMemGetInfo(&memfree,&memtotal); + if(dev==0) *mem_GPU_global=memfree; + if(memfree -#include -#include -#include "voxel_backprojection2.hpp" -#include "TIGRE_common.hpp" -#include -#include "GpuIds.hpp" - -// https://stackoverflow.com/questions/16282136/is-there-a-cuda-equivalent-of-perror -#define cudaCheckErrors(msg) \ -do { \ - cudaError_t __err = cudaGetLastError(); \ - if (__err != cudaSuccess) { \ - mexPrintf("%s \n",msg);\ - mexErrMsgIdAndTxt("CBCT:CUDA:Atb",cudaGetErrorString(__err));\ - } \ -} while (0) - - -#define MAXTREADS 1024 - /*GEOMETRY DEFINITION - * - * Detector plane, behind - * |-----------------------------| - * | | - * | | - * | | - * | | - * | +--------+ | - * | / /| | - * A Z | / / |*D | - * | | +--------+ | | - * | | | | | | - * | | | *O | + | - * *--->y | | | / | - * / | | |/ | - * V X | +--------+ | - * |-----------------------------| - * - * *S - * - * - * - * - * - **/ - -// this definitionmust go here. -void CreateTexture2(const GpuIds& gpuids, float* projectiondata,Geometry geo,cudaArray** d_cuArrTex,unsigned int nangles, cudaTextureObject_t *texImage,cudaStream_t* stream,int nStreamDevice,bool allocate); - -__global__ void matrixConstantMultiply(const Geometry geo,float* image,float constant){ - size_t idx = threadIdx.x + blockIdx.x * blockDim.x; - for(; idx=geo.nVoxelX || indY>=geo.nVoxelY || startIndZ>=geo.nVoxelZ) - return; - - // We'll keep a local auxiliary array of values of a column of voxels that this thread will update - float voxelColumn[VOXELS_PER_THREAD]; - - // First we need to copy the curent 3D volume values from the column to our auxiliary array so that we can then - // work on them (update them by computing values from multiple projections) locally - avoiding main memory reads/writes - - unsigned long colIdx; -#pragma unroll - for(colIdx=0; colIdx=geo.nVoxelZ) - break; // break the loop. - - unsigned long long idx =indZ*(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY+indY*(unsigned long long)geo.nVoxelX + indX; - voxelColumn[colIdx] = image[idx]; // Read the current volume value that we'll update by computing values from MULTIPLE projections (not just one) - // We'll be updating the local (register) variable, avoiding reads/writes from the slow main memory. - } // END copy 3D volume voxels to local array - - // Now iterate through projections -#pragma unroll - for(unsigned long projNumber=0; projNumber=totalNoOfProjections) - break; - - Point3D deltaX = projParamsArray2Dev[7*projNumber]; // 6*projNumber because we have 6 Point3D values per projection - Point3D deltaY = projParamsArray2Dev[7*projNumber+1]; - Point3D deltaZ = projParamsArray2Dev[7*projNumber+2]; - Point3D xyzOrigin = projParamsArray2Dev[7*projNumber+3]; - Point3D xyzOffset = projParamsArray2Dev[7*projNumber+4]; - Point3D uv0Offset = projParamsArray2Dev[7*projNumber+5]; - Point3D S = projParamsArray2Dev[7*projNumber+6]; - - float sinalpha = projSinCosArray2Dev[5*projNumber]; // 2*projNumber because we have 2 float (sin or cos angle) values per projection - float cosalpha = projSinCosArray2Dev[5*projNumber+1]; - float COR = projSinCosArray2Dev[5*projNumber+2]; - float DSD = projSinCosArray2Dev[5*projNumber+3]; - float DSO = projSinCosArray2Dev[5*projNumber+4]; - // Precomputations for the weights: - //Real coords of Source - // We already have S.x (geo.DSO), and S.y and S.z are always zero. we just need to rotate - Point3D realS; - realS.x= DSO*cosalpha; - realS.y=-DSO*sinalpha; - realS.z=0; - - - Point3D realvoxel_init; - realvoxel_init.x=-geo.sVoxelX/2+geo.dVoxelX/2+xyzOffset.x; - realvoxel_init.y=-geo.sVoxelY/2+geo.dVoxelY/2+xyzOffset.y; - realvoxel_init.z=-geo.sVoxelZ/2+geo.dVoxelZ/2+xyzOffset.z; - // Real XYZ coordinates of Detector. - Point3D realD, realDaux; - // We know the index of the detector (u,v). Start from there. - realDaux.x=-(DSD-DSO); - - // Now iterate through Z in our voxel column FOR A GIVEN PROJECTION -#pragma unroll - for(colIdx=0; colIdx=geo.nVoxelZ) - break; // break the loop. - - // "XYZ" in the scaled coordinate system of the current point. The image is rotated with the projection angles. - Point3D P; - P.x=(xyzOrigin.x+indX*deltaX.x+indY*deltaY.x+indZ*deltaZ.x); - P.y=(xyzOrigin.y+indX*deltaX.y+indY*deltaY.y+indZ*deltaZ.y)-COR/geo.dDetecU; - P.z=(xyzOrigin.z+indX*deltaX.z+indY*deltaY.z+indZ*deltaZ.z); - - // This is the vector defining the line from the source to the Voxel - float vectX,vectY,vectZ; - vectX=(P.x -S.x); - vectY=(P.y -S.y); - vectZ=(P.z -S.z); - - // Get the coordinates in the detector UV where the mid point of the voxel is projected. - float t=__fdividef(DSO-DSD-S.x,vectX); - float y,z; - y=vectY*t+S.y; - z=vectZ*t+S.z; - float u,v; - u=y+(float)geo.nDetecU*0.5f; - v=z+(float)geo.nDetecV*0.5f; -#if IS_FOR_MATLAB_TIGRE - float sample=tex3D(tex, v, u ,indAlpha+0.5f); -#else - float sample=tex3D(tex, u, v ,indAlpha+0.5f); -#endif - float weight=0; - // - // - // - // IMPORTANT: The weights are almost 50% of the computational time. Is there a way of speeding this up?? - // - //Real coordinates of Voxel. Instead of reverting the transformation, its less math (faster) to compute it from the indexes. - Point3D realvoxel; - - realvoxel.x=realvoxel_init.x+indX*geo.dVoxelX; - realvoxel.y=realvoxel_init.y+indY*geo.dVoxelY; - realvoxel.z=realvoxel_init.z+indZ*geo.dVoxelZ; - - - - realDaux.y=(-geo.sDetecU+geo.dDetecU)*0.5f + u*geo.dDetecU +uv0Offset.x; - realD.z =(-geo.sDetecV+geo.dDetecV)*0.5f + v*geo.dDetecV +uv0Offset.y; - //rotate the detector - realD.x= realDaux.x*cosalpha + realDaux.y*sinalpha; //sin(-x)=-sin(x) , cos(-x)=cos(x) - realD.y=-realDaux.x*sinalpha + realDaux.y*cosalpha; //sin(-x)=-sin(x) , cos(-x)=cos(x) - float L,lsq; - - L = __fsqrt_rd( (realS.x-realD.x)*(realS.x-realD.x)+ (realS.y-realD.y)*(realS.y-realD.y)+ (realD.z)*(realD.z)); // Sz=0 always. - lsq = (realS.x-realvoxel.x)*(realS.x-realvoxel.x) - + (realS.y-realvoxel.y)*(realS.y-realvoxel.y) - + (realS.z-realvoxel.z)*(realS.z-realvoxel.z); - - weight=__fdividef(L*L*L,(DSD*lsq)); -// weight=1; - // Get Value in the computed (U,V) and multiply by the corresponding weight. - // indAlpha is the ABSOLUTE number of projection in the projection array (NOT the current number of projection set!) - voxelColumn[colIdx]+=sample* weight; - } // END iterating through column of voxels - - } // END iterating through multiple projections - - // And finally copy the updated local voxelColumn array back to our 3D volume (main memory) -#pragma unroll - for(colIdx=0; colIdx=geo.nVoxelZ) - break; // break the loop. - - unsigned long long idx =indZ*(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY+indY*(unsigned long long)geo.nVoxelX + indX; - image[idx] = voxelColumn[colIdx]; // Read the current volume value that we'll update by computing values from MULTIPLE projections (not just one) - // We'll be updating the local (register) variable, avoiding reads/writes from the slow main memory. - // According to references (Papenhausen), doing = is better than +=, since += requires main memory read followed by a write. - // We did all the reads into the local array at the BEGINNING of this kernel. According to Papenhausen, this type of read-write split is - // better for avoiding memory congestion. - } // END copy updated voxels from local array to our 3D volume - -} // END kernelPixelBackprojectionFDK - - - - -//______________________________________________________________________________ -// -// Function: voxel_backprojection -// -// Description: Main host function for FDK backprojection (invokes the kernel) -//______________________________________________________________________________ - -int voxel_backprojection2(float * projections, Geometry geo, float* result,float const * const alphas, int nalpha, const GpuIds& gpuids){ - - - - - // Prepare for MultiGPU - int deviceCount = gpuids.GetLength(); - cudaCheckErrors("Device query fail"); - if (deviceCount == 0) { - mexErrMsgIdAndTxt("Atb:Voxel_backprojection:GPUselect","There are no available device(s) that support CUDA\n"); - } - - - // CODE assumes - // 1.-All available devices are usable by this code - // 2.-All available devices are equal, they are the same machine (warning thrown) - // Check the available devices, and if they are the same - if (!gpuids.AreEqualDevices()) { - mexWarnMsgIdAndTxt("Atb:Voxel_backprojection2:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); - } - - int dev; - - - // Split the CT problem - unsigned int split_image; - unsigned int split_projections; - splitCTbackprojection(gpuids,geo,nalpha,&split_image,&split_projections); - - - // Create the arrays for the geometry. The main difference is that geo.offZ has been tuned for the - // image slices. The rest of the Geometry is the same - Geometry* geoArray=(Geometry*)malloc(split_image*deviceCount*sizeof(Geometry)); - createGeoArray(split_image*deviceCount,geo,geoArray,nalpha); - - // Now lest allocate all the image memory on the GPU, so we can use it later. If we have made our numbers correctly - // in the previous section this should leave enough space for the textures. - size_t num_bytes_img = (size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geoArray[0].nVoxelZ* sizeof(float); - float** dimage=(float**)malloc(deviceCount*sizeof(float*)); - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaMalloc((void**)&dimage[dev], num_bytes_img); - cudaCheckErrors("cudaMalloc fail"); - } - - - //Pagelock memory for synchronous copy. - // Lets try to make the host memory pinned: - // We laredy queried the GPU and assuemd they are the same, thus should have the same attributes. - int isHostRegisterSupported = 0; -#if CUDART_VERSION >= 9020 - cudaDeviceGetAttribute(&isHostRegisterSupported,cudaDevAttrHostRegisterSupported,gpuids[0]); -#endif - // empirical testing shows that when the image split is smaller than 1 (also implies the image is not very big), the time to - // pin the memory is greater than the lost time in Synchronously launching the memcpys. This is only worth it when the image is too big. - if (isHostRegisterSupported & split_image>1){ - cudaHostRegister(result, (size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geo.nVoxelZ*(size_t)sizeof(float),cudaHostRegisterPortable); - } - if (isHostRegisterSupported ){ - cudaHostRegister(projections, (size_t)geo.nDetecU*(size_t)geo.nDetecV*(size_t)nalpha*(size_t)sizeof(float),cudaHostRegisterPortable); - } - cudaCheckErrors("Error pinning memory"); - - - - - - //If it is the first time, lets make sure our image is zeroed. - int nStreamDevice=2; - int nStreams=deviceCount*nStreamDevice; - cudaStream_t* stream=(cudaStream_t*)malloc(nStreams*sizeof(cudaStream_t));; - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - for (int i = 0; i < nStreamDevice; ++i){ - cudaStreamCreate(&stream[i+dev*nStreamDevice]); - - } - } - - // Kernel auxiliary variables - Point3D* projParamsArray2Host; - cudaMallocHost((void**)&projParamsArray2Host,7*PROJ_PER_KERNEL*sizeof(Point3D)); - float* projSinCosArray2Host; - cudaMallocHost((void**)&projSinCosArray2Host,5*PROJ_PER_KERNEL*sizeof(float)); - - // Texture object variables - cudaTextureObject_t *texProj; - cudaArray **d_cuArrTex; - texProj =(cudaTextureObject_t*)malloc(deviceCount*2*sizeof(cudaTextureObject_t)); - d_cuArrTex =(cudaArray**)malloc(deviceCount*2*sizeof(cudaArray*)); - - - - unsigned int proj_split_overlap_number; - // Start with the main loop. The Projection data needs to be allocated and dealocated in the main loop - // as due to the nature of cudaArrays, we can not reuse them. This should not be a problem for the fast execution - // of the code, as repeated allocation and deallocation only happens when the projection data is very very big, - // and therefore allcoation time should be negligible, fluctuation of other computations should mask the time. - unsigned long long proj_linear_idx_start; - unsigned int current_proj_split_size,current_proj_overlap_split_size; - size_t num_bytes_img_curr; - size_t img_linear_idx_start; - float** partial_projection; - size_t* proj_split_size; - - for(unsigned int img_slice=0;img_slice=proj_split_size[proj_block_split]) - break; // Exit the loop. Even when we leave the param arrays only partially filled, this is OK, since the kernel will check bounds anyway. - if(currProjNumber_global>=nalpha) - break; // Exit the loop. Even when we leave the param arrays only partially filled, this is OK, since the kernel will check bounds anyway. - - Point3D deltaX,deltaY,deltaZ,xyzOrigin, offOrig, offDetec,source; - float sinalpha,cosalpha; - - geoArray[img_slice*deviceCount+dev].alpha=-alphas[currProjNumber_global*3];//we got 3 angles now. - geoArray[img_slice*deviceCount+dev].theta=-alphas[currProjNumber_global*3+1]; - geoArray[img_slice*deviceCount+dev].psi =-alphas[currProjNumber_global*3+2]; - - sinalpha=sin(geoArray[img_slice*deviceCount+dev].alpha); - cosalpha=cos(geoArray[img_slice*deviceCount+dev].alpha); - - projSinCosArray2Host[5*j]=sinalpha; // 2*j because we have 2 float (sin or cos angle) values per projection - projSinCosArray2Host[5*j+1]=cosalpha; - projSinCosArray2Host[5*j+2]=geo.COR[currProjNumber_global]; - projSinCosArray2Host[5*j+3]=geo.DSD[currProjNumber_global]; - projSinCosArray2Host[5*j+4]=geo.DSO[currProjNumber_global]; - - computeDeltasCube(geoArray[img_slice*deviceCount+dev],currProjNumber_global,&xyzOrigin,&deltaX,&deltaY,&deltaZ,&source); - - offOrig.x=geo.offOrigX[currProjNumber_global]; - offOrig.y=geo.offOrigY[currProjNumber_global]; - offOrig.z=geoArray[img_slice*deviceCount+dev].offOrigZ[currProjNumber_global]; - - offDetec.x=geo.offDetecU[currProjNumber_global]; - offDetec.y=geo.offDetecV[currProjNumber_global]; - offDetec.z=0;//unused - - projParamsArray2Host[7*j] =deltaX; // 7*j because we have 7 Point3D values per projection - projParamsArray2Host[7*j+1]=deltaY; - projParamsArray2Host[7*j+2]=deltaZ; - projParamsArray2Host[7*j+3]=xyzOrigin; - projParamsArray2Host[7*j+4]=offOrig; - projParamsArray2Host[7*j+5]=offDetec; - projParamsArray2Host[7*j+6]=source; - - } // END for (preparing params for kernel call) - - // Copy the prepared parameter arrays to constant memory to make it available for the kernel - cudaMemcpyToSymbolAsync(projSinCosArray2Dev, projSinCosArray2Host, sizeof(float)*5*PROJ_PER_KERNEL,0,cudaMemcpyHostToDevice,stream[dev*nStreamDevice]); - cudaMemcpyToSymbolAsync(projParamsArray2Dev, projParamsArray2Host, sizeof(Point3D)*7*PROJ_PER_KERNEL,0,cudaMemcpyHostToDevice,stream[dev*nStreamDevice]); - cudaStreamSynchronize(stream[dev*nStreamDevice]); - kernelPixelBackprojection<<>>(geoArray[img_slice*deviceCount+dev],dimage[dev],i,proj_split_size[proj_block_split],texProj[(proj_block_split%2)*deviceCount+dev]); - - } // END for - ////////////////////////////////////////////////////////////////////////////////////// - // END RB code, Main reconstruction loop: go through projections (rotation angles) and backproject - ////////////////////////////////////////////////////////////////////////////////////// - } - } // END sub-split of current projection chunk - - } // END projection splits - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - matrixConstantMultiply<<<60,MAXTREADS,0,stream[dev*nStreamDevice]>>>( geoArray[img_slice*deviceCount+dev],dimage[dev],geo.dVoxelX*geo.dVoxelY*geo.dVoxelZ/(geo.dDetecU*geo.dDetecV)); - } - - // Now we need to take the image out of the GPU - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaStreamSynchronize(stream[dev*nStreamDevice]); - - num_bytes_img_curr=(size_t)geoArray[img_slice*deviceCount+dev].nVoxelX*(size_t)geoArray[img_slice*deviceCount+dev].nVoxelY*(size_t)geoArray[img_slice*deviceCount+dev].nVoxelZ*sizeof(float); - img_linear_idx_start=(size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geoArray[0].nVoxelZ*(size_t)(img_slice*deviceCount+dev); - cudaMemcpyAsync(&result[img_linear_idx_start], dimage[dev], num_bytes_img_curr, cudaMemcpyDeviceToHost,stream[dev*nStreamDevice+1]); - } - } // end image splits - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDeviceSynchronize(); - } - - - // Clean the GPU - bool two_buffers_used=((((nalpha+split_projections-1)/split_projections)+PROJ_PER_KERNEL-1)/PROJ_PER_KERNEL)>1; - for(unsigned int i=0; i<2;i++){ // 2 buffers (if needed, maybe only 1) - if (!two_buffers_used && i==1) - break; for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaDestroyTextureObject(texProj[i*deviceCount+dev]); - cudaFreeArray(d_cuArrTex[i*deviceCount+dev]); - } - } - free(d_cuArrTex); - free(texProj); - - for (dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaFree(dimage[dev]); - } - free(dimage); - - cudaFreeHost(projSinCosArray2Host); - cudaFreeHost(projParamsArray2Host); - free(partial_projection); - free(proj_split_size); - - freeGeoArray(split_image*deviceCount,geoArray); -#ifndef NO_PINNED_MEMORY - if (isHostRegisterSupported & split_image>1){ - cudaHostUnregister(result); - } - if (isHostRegisterSupported){ - cudaHostUnregister(projections); - } -#endif - for (int i = 0; i < nStreams; ++i) - cudaStreamDestroy(stream[i]); - - cudaCheckErrors("cudaFree fail"); - -// cudaDeviceReset(); // For the Nvidia Visual Profiler - return 0; - -} // END voxel_backprojection - - - - - -void CreateTexture2(const GpuIds& gpuids, float* projectiondata,Geometry geo,cudaArray** d_cuArrTex,unsigned int nangles, cudaTextureObject_t *texImage,cudaStream_t* stream,int nStreamDevice,bool allocate){ - //size_t size_image=geo.nVoxelX*geo.nVoxelY*geo.nVoxelZ; - int num_devices = gpuids.GetLength(); -#if IS_FOR_MATLAB_TIGRE - const cudaExtent extent =make_cudaExtent(geo.nDetecV, geo.nDetecU, nangles); -#else - const cudaExtent extent =make_cudaExtent(geo.nDetecU, geo.nDetecV, nangles); -#endif - if (allocate){ - for (unsigned int dev = 0; dev < num_devices; dev++){ - cudaSetDevice(gpuids[dev]); - - //cudaArray Descriptor - cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(); - //cuda Array - cudaMalloc3DArray(&d_cuArrTex[dev], &channelDesc, extent); - - } - } - for (unsigned int dev = 0; dev < num_devices; dev++){ - cudaSetDevice(gpuids[dev]); - cudaMemcpy3DParms copyParams = {0}; - //Array creation - copyParams.srcPtr = make_cudaPitchedPtr((void *)projectiondata, extent.width*sizeof(float), extent.width, extent.height); - copyParams.dstArray = d_cuArrTex[dev]; - copyParams.extent = extent; - copyParams.kind = cudaMemcpyHostToDevice; - cudaMemcpy3DAsync(©Params,stream[dev*nStreamDevice+1]); - } - - //Array creation End - for (unsigned int dev = 0; dev < num_devices; dev++){ - cudaSetDevice(gpuids[dev]); - cudaResourceDesc texRes; - memset(&texRes, 0, sizeof(cudaResourceDesc)); - texRes.resType = cudaResourceTypeArray; - texRes.res.array.array = d_cuArrTex[dev]; - cudaTextureDesc texDescr; - memset(&texDescr, 0, sizeof(cudaTextureDesc)); - texDescr.normalizedCoords = false; - texDescr.filterMode = cudaFilterModeLinear; - texDescr.addressMode[0] = cudaAddressModeBorder; - texDescr.addressMode[1] = cudaAddressModeBorder; - texDescr.addressMode[2] = cudaAddressModeBorder; - texDescr.readMode = cudaReadModeElementType; - cudaCreateTextureObject(&texImage[dev], &texRes, &texDescr, NULL); - } -} -#ifndef BACKPROJECTION_HPP -void splitCTbackprojection(const GpuIds& gpuids, Geometry geo,int nalpha, unsigned int* split_image, unsigned int * split_projections){ - - - // We don't know if the devices are being used. lets check that. and only use the amount of memory we need. - - size_t mem_GPU_global; - checkFreeMemory(gpuids, &mem_GPU_global); - const int deviceCount = gpuids.GetLength(); - - // Compute how much memory each of the relevant memory pieces need - size_t mem_image= (unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY*(unsigned long long)geo.nVoxelZ*sizeof(float); - size_t mem_proj= (unsigned long long)geo.nDetecU*(unsigned long long)geo.nDetecV*sizeof(float); - - - - - // Does everything fit in the GPU? - - if(mem_image/deviceCount+mem_proj*PROJ_PER_KERNEL*2x=Px.x-P.x; deltaX->y=Px.y-P.y; deltaX->z=Px.z-P.z; - deltaY->x=Py.x-P.x; deltaY->y=Py.y-P.y; deltaY->z=Py.z-P.z; - deltaZ->x=Pz.x-P.x; deltaZ->y=Pz.y-P.y; deltaZ->z=Pz.z-P.z; - - - *xyzorigin=P.to_float(); - *S=source.to_float(); -} // END computeDeltasCube - -void checkFreeMemory(const GpuIds& gpuids,size_t *mem_GPU_global){ - size_t memfree; - size_t memtotal; - const int gpuids.GetLength(); - - for (int dev = 0; dev < deviceCount; dev++){ - cudaSetDevice(gpuids[dev]); - cudaMemGetInfo(&memfree,&memtotal); - if(dev==0) *mem_GPU_global=memfree; - if(memfree +#include +#include +#include "voxel_backprojection2.hpp" +#include "TIGRE_common.hpp" +#include +#include "GpuIds.hpp" + +// https://stackoverflow.com/questions/16282136/is-there-a-cuda-equivalent-of-perror +#define hipCheckErrors(msg) \ +do { \ + hipError_t __err = hipGetLastError(); \ + if (__err != hipSuccess) { \ + mexPrintf("%s \n",msg);\ + mexErrMsgIdAndTxt("CBCT:CUDA:Atb",hipGetErrorString(__err));\ + } \ +} while (0) + + +#define MAXTREADS 1024 + /*GEOMETRY DEFINITION + * + * Detector plane, behind + * |-----------------------------| + * | | + * | | + * | | + * | | + * | +--------+ | + * | / /| | + * A Z | / / |*D | + * | | +--------+ | | + * | | | | | | + * | | | *O | + | + * *--->y | | | / | + * / | | |/ | + * V X | +--------+ | + * |-----------------------------| + * + * *S + * + * + * + * + * + **/ + +// this definitionmust go here. +void CreateTexture2(const GpuIds& gpuids, float* projectiondata,Geometry geo,hipArray** d_cuArrTex,unsigned int nangles, hipTextureObject_t *texImage,hipStream_t* stream,int nStreamDevice,bool allocate); + +__global__ void matrixConstantMultiply(const Geometry geo,float* image,float constant){ + size_t idx = threadIdx.x + blockIdx.x * blockDim.x; + for(; idx=geo.nVoxelX || indY>=geo.nVoxelY || startIndZ>=geo.nVoxelZ) + return; + + // We'll keep a local auxiliary array of values of a column of voxels that this thread will update + float voxelColumn[VOXELS_PER_THREAD]; + + // First we need to copy the curent 3D volume values from the column to our auxiliary array so that we can then + // work on them (update them by computing values from multiple projections) locally - avoiding main memory reads/writes + + unsigned long colIdx; +#pragma unroll + for(colIdx=0; colIdx=geo.nVoxelZ) + break; // break the loop. + + unsigned long long idx =indZ*(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY+indY*(unsigned long long)geo.nVoxelX + indX; + voxelColumn[colIdx] = image[idx]; // Read the current volume value that we'll update by computing values from MULTIPLE projections (not just one) + // We'll be updating the local (register) variable, avoiding reads/writes from the slow main memory. + } // END copy 3D volume voxels to local array + + // Now iterate through projections +#pragma unroll + for(unsigned long projNumber=0; projNumber=totalNoOfProjections) + break; + + Point3D deltaX = projParamsArray2Dev[7*projNumber]; // 6*projNumber because we have 6 Point3D values per projection + Point3D deltaY = projParamsArray2Dev[7*projNumber+1]; + Point3D deltaZ = projParamsArray2Dev[7*projNumber+2]; + Point3D xyzOrigin = projParamsArray2Dev[7*projNumber+3]; + Point3D xyzOffset = projParamsArray2Dev[7*projNumber+4]; + Point3D uv0Offset = projParamsArray2Dev[7*projNumber+5]; + Point3D S = projParamsArray2Dev[7*projNumber+6]; + + float sinalpha = projSinCosArray2Dev[5*projNumber]; // 2*projNumber because we have 2 float (sin or cos angle) values per projection + float cosalpha = projSinCosArray2Dev[5*projNumber+1]; + float COR = projSinCosArray2Dev[5*projNumber+2]; + float DSD = projSinCosArray2Dev[5*projNumber+3]; + float DSO = projSinCosArray2Dev[5*projNumber+4]; + // Precomputations for the weights: + //Real coords of Source + // We already have S.x (geo.DSO), and S.y and S.z are always zero. we just need to rotate + Point3D realS; + realS.x= DSO*cosalpha; + realS.y=-DSO*sinalpha; + realS.z=0; + + + Point3D realvoxel_init; + realvoxel_init.x=-geo.sVoxelX/2+geo.dVoxelX/2+xyzOffset.x; + realvoxel_init.y=-geo.sVoxelY/2+geo.dVoxelY/2+xyzOffset.y; + realvoxel_init.z=-geo.sVoxelZ/2+geo.dVoxelZ/2+xyzOffset.z; + // Real XYZ coordinates of Detector. + Point3D realD, realDaux; + // We know the index of the detector (u,v). Start from there. + realDaux.x=-(DSD-DSO); + + // Now iterate through Z in our voxel column FOR A GIVEN PROJECTION +#pragma unroll + for(colIdx=0; colIdx=geo.nVoxelZ) + break; // break the loop. + + // "XYZ" in the scaled coordinate system of the current point. The image is rotated with the projection angles. + Point3D P; + P.x=(xyzOrigin.x+indX*deltaX.x+indY*deltaY.x+indZ*deltaZ.x); + P.y=(xyzOrigin.y+indX*deltaX.y+indY*deltaY.y+indZ*deltaZ.y)-COR/geo.dDetecU; + P.z=(xyzOrigin.z+indX*deltaX.z+indY*deltaY.z+indZ*deltaZ.z); + + // This is the vector defining the line from the source to the Voxel + float vectX,vectY,vectZ; + vectX=(P.x -S.x); + vectY=(P.y -S.y); + vectZ=(P.z -S.z); + + // Get the coordinates in the detector UV where the mid point of the voxel is projected. + float t=__fdividef(DSO-DSD-S.x,vectX); + float y,z; + y=vectY*t+S.y; + z=vectZ*t+S.z; + float u,v; + u=y+(float)geo.nDetecU*0.5f; + v=z+(float)geo.nDetecV*0.5f; +#if IS_FOR_MATLAB_TIGRE + float sample=tex3D(tex, v, u ,indAlpha+0.5f); +#else + float sample=tex3D(tex, u, v ,indAlpha+0.5f); +#endif + float weight=0; + // + // + // + // IMPORTANT: The weights are almost 50% of the computational time. Is there a way of speeding this up?? + // + //Real coordinates of Voxel. Instead of reverting the transformation, its less math (faster) to compute it from the indexes. + Point3D realvoxel; + + realvoxel.x=realvoxel_init.x+indX*geo.dVoxelX; + realvoxel.y=realvoxel_init.y+indY*geo.dVoxelY; + realvoxel.z=realvoxel_init.z+indZ*geo.dVoxelZ; + + + + realDaux.y=(-geo.sDetecU+geo.dDetecU)*0.5f + u*geo.dDetecU +uv0Offset.x; + realD.z =(-geo.sDetecV+geo.dDetecV)*0.5f + v*geo.dDetecV +uv0Offset.y; + //rotate the detector + realD.x= realDaux.x*cosalpha + realDaux.y*sinalpha; //sin(-x)=-sin(x) , cos(-x)=cos(x) + realD.y=-realDaux.x*sinalpha + realDaux.y*cosalpha; //sin(-x)=-sin(x) , cos(-x)=cos(x) + float L,lsq; + + L = __fsqrt_rd( (realS.x-realD.x)*(realS.x-realD.x)+ (realS.y-realD.y)*(realS.y-realD.y)+ (realD.z)*(realD.z)); // Sz=0 always. + lsq = (realS.x-realvoxel.x)*(realS.x-realvoxel.x) + + (realS.y-realvoxel.y)*(realS.y-realvoxel.y) + + (realS.z-realvoxel.z)*(realS.z-realvoxel.z); + + weight=__fdividef(L*L*L,(DSD*lsq)); +// weight=1; + // Get Value in the computed (U,V) and multiply by the corresponding weight. + // indAlpha is the ABSOLUTE number of projection in the projection array (NOT the current number of projection set!) + voxelColumn[colIdx]+=sample* weight; + } // END iterating through column of voxels + + } // END iterating through multiple projections + + // And finally copy the updated local voxelColumn array back to our 3D volume (main memory) +#pragma unroll + for(colIdx=0; colIdx=geo.nVoxelZ) + break; // break the loop. + + unsigned long long idx =indZ*(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY+indY*(unsigned long long)geo.nVoxelX + indX; + image[idx] = voxelColumn[colIdx]; // Read the current volume value that we'll update by computing values from MULTIPLE projections (not just one) + // We'll be updating the local (register) variable, avoiding reads/writes from the slow main memory. + // According to references (Papenhausen), doing = is better than +=, since += requires main memory read followed by a write. + // We did all the reads into the local array at the BEGINNING of this kernel. According to Papenhausen, this type of read-write split is + // better for avoiding memory congestion. + } // END copy updated voxels from local array to our 3D volume + +} // END kernelPixelBackprojectionFDK + + + + +//______________________________________________________________________________ +// +// Function: voxel_backprojection +// +// Description: Main host function for FDK backprojection (invokes the kernel) +//______________________________________________________________________________ + +int voxel_backprojection2(float * projections, Geometry geo, float* result,float const * const alphas, int nalpha, const GpuIds& gpuids){ + + + + + // Prepare for MultiGPU + int deviceCount = gpuids.GetLength(); + hipCheckErrors("Device query fail"); + if (deviceCount == 0) { + mexErrMsgIdAndTxt("Atb:Voxel_backprojection:GPUselect","There are no available device(s) that support CUDA\n"); + } + + + // CODE assumes + // 1.-All available devices are usable by this code + // 2.-All available devices are equal, they are the same machine (warning thrown) + // Check the available devices, and if they are the same + if (!gpuids.AreEqualDevices()) { + mexWarnMsgIdAndTxt("Atb:Voxel_backprojection2:GPUselect","Detected one (or more) different GPUs.\n This code is not smart enough to separate the memory GPU wise if they have different computational times or memory limits.\n First GPU parameters used. If the code errors you might need to change the way GPU selection is performed."); + } + + int dev; + + + // Split the CT problem + unsigned int split_image; + unsigned int split_projections; + splitCTbackprojection(gpuids,geo,nalpha,&split_image,&split_projections); + + + // Create the arrays for the geometry. The main difference is that geo.offZ has been tuned for the + // image slices. The rest of the Geometry is the same + Geometry* geoArray=(Geometry*)malloc(split_image*deviceCount*sizeof(Geometry)); + createGeoArray(split_image*deviceCount,geo,geoArray,nalpha); + + // Now lest allocate all the image memory on the GPU, so we can use it later. If we have made our numbers correctly + // in the previous section this should leave enough space for the textures. + size_t num_bytes_img = (size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geoArray[0].nVoxelZ* sizeof(float); + float** dimage=(float**)malloc(deviceCount*sizeof(float*)); + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipMalloc((void**)&dimage[dev], num_bytes_img); + hipCheckErrors("hipMalloc fail"); + } + + + //Pagelock memory for synchronous copy. + // Lets try to make the host memory pinned: + // We laredy queried the GPU and assuemd they are the same, thus should have the same attributes. + int isHostRegisterSupported = 0; +#if CUDART_VERSION >= 9020 + hipDeviceGetAttribute(&isHostRegisterSupported,hipDevAttrHostRegisterSupported,gpuids[0]); +#endif + // empirical testing shows that when the image split is smaller than 1 (also implies the image is not very big), the time to + // pin the memory is greater than the lost time in Synchronously launching the memcpys. This is only worth it when the image is too big. + if (isHostRegisterSupported & split_image>1){ + hipHostRegister(result, (size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geo.nVoxelZ*(size_t)sizeof(float),hipHostRegisterPortable); + } + if (isHostRegisterSupported ){ + hipHostRegister(projections, (size_t)geo.nDetecU*(size_t)geo.nDetecV*(size_t)nalpha*(size_t)sizeof(float),hipHostRegisterPortable); + } + hipCheckErrors("Error pinning memory"); + + + + + + //If it is the first time, lets make sure our image is zeroed. + int nStreamDevice=2; + int nStreams=deviceCount*nStreamDevice; + hipStream_t* stream=(hipStream_t*)malloc(nStreams*sizeof(hipStream_t));; + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + for (int i = 0; i < nStreamDevice; ++i){ + hipStreamCreate(&stream[i+dev*nStreamDevice]); + + } + } + + // Kernel auxiliary variables + Point3D* projParamsArray2Host; + hipMallocHost((void**)&projParamsArray2Host,7*PROJ_PER_KERNEL*sizeof(Point3D)); + float* projSinCosArray2Host; + hipMallocHost((void**)&projSinCosArray2Host,5*PROJ_PER_KERNEL*sizeof(float)); + + // Texture object variables + hipTextureObject_t *texProj; + hipArray **d_cuArrTex; + texProj =(hipTextureObject_t*)malloc(deviceCount*2*sizeof(hipTextureObject_t)); + d_cuArrTex =(hipArray**)malloc(deviceCount*2*sizeof(hipArray*)); + + + + unsigned int proj_split_overlap_number; + // Start with the main loop. The Projection data needs to be allocated and dealocated in the main loop + // as due to the nature of hipArrays, we can not reuse them. This should not be a problem for the fast execution + // of the code, as repeated allocation and deallocation only happens when the projection data is very very big, + // and therefore allcoation time should be negligible, fluctuation of other computations should mask the time. + unsigned long long proj_linear_idx_start; + unsigned int current_proj_split_size,current_proj_overlap_split_size; + size_t num_bytes_img_curr; + size_t img_linear_idx_start; + float** partial_projection; + size_t* proj_split_size; + + for(unsigned int img_slice=0;img_slice=proj_split_size[proj_block_split]) + break; // Exit the loop. Even when we leave the param arrays only partially filled, this is OK, since the kernel will check bounds anyway. + if(currProjNumber_global>=nalpha) + break; // Exit the loop. Even when we leave the param arrays only partially filled, this is OK, since the kernel will check bounds anyway. + + Point3D deltaX,deltaY,deltaZ,xyzOrigin, offOrig, offDetec,source; + float sinalpha,cosalpha; + + geoArray[img_slice*deviceCount+dev].alpha=-alphas[currProjNumber_global*3];//we got 3 angles now. + geoArray[img_slice*deviceCount+dev].theta=-alphas[currProjNumber_global*3+1]; + geoArray[img_slice*deviceCount+dev].psi =-alphas[currProjNumber_global*3+2]; + + sinalpha=sin(geoArray[img_slice*deviceCount+dev].alpha); + cosalpha=cos(geoArray[img_slice*deviceCount+dev].alpha); + + projSinCosArray2Host[5*j]=sinalpha; // 2*j because we have 2 float (sin or cos angle) values per projection + projSinCosArray2Host[5*j+1]=cosalpha; + projSinCosArray2Host[5*j+2]=geo.COR[currProjNumber_global]; + projSinCosArray2Host[5*j+3]=geo.DSD[currProjNumber_global]; + projSinCosArray2Host[5*j+4]=geo.DSO[currProjNumber_global]; + + computeDeltasCube(geoArray[img_slice*deviceCount+dev],currProjNumber_global,&xyzOrigin,&deltaX,&deltaY,&deltaZ,&source); + + offOrig.x=geo.offOrigX[currProjNumber_global]; + offOrig.y=geo.offOrigY[currProjNumber_global]; + offOrig.z=geoArray[img_slice*deviceCount+dev].offOrigZ[currProjNumber_global]; + + offDetec.x=geo.offDetecU[currProjNumber_global]; + offDetec.y=geo.offDetecV[currProjNumber_global]; + offDetec.z=0;//unused + + projParamsArray2Host[7*j] =deltaX; // 7*j because we have 7 Point3D values per projection + projParamsArray2Host[7*j+1]=deltaY; + projParamsArray2Host[7*j+2]=deltaZ; + projParamsArray2Host[7*j+3]=xyzOrigin; + projParamsArray2Host[7*j+4]=offOrig; + projParamsArray2Host[7*j+5]=offDetec; + projParamsArray2Host[7*j+6]=source; + + } // END for (preparing params for kernel call) + + // Copy the prepared parameter arrays to constant memory to make it available for the kernel + hipMemcpyToSymbolAsync(projSinCosArray2Dev, projSinCosArray2Host, sizeof(float)*5*PROJ_PER_KERNEL,0,hipMemcpyHostToDevice,stream[dev*nStreamDevice]); + hipMemcpyToSymbolAsync(projParamsArray2Dev, projParamsArray2Host, sizeof(Point3D)*7*PROJ_PER_KERNEL,0,hipMemcpyHostToDevice,stream[dev*nStreamDevice]); + hipStreamSynchronize(stream[dev*nStreamDevice]); + kernelPixelBackprojection<<>>(geoArray[img_slice*deviceCount+dev],dimage[dev],i,proj_split_size[proj_block_split],texProj[(proj_block_split%2)*deviceCount+dev]); + + } // END for + ////////////////////////////////////////////////////////////////////////////////////// + // END RB code, Main reconstruction loop: go through projections (rotation angles) and backproject + ////////////////////////////////////////////////////////////////////////////////////// + } + } // END sub-split of current projection chunk + + } // END projection splits + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + matrixConstantMultiply<<<60,MAXTREADS,0,stream[dev*nStreamDevice]>>>( geoArray[img_slice*deviceCount+dev],dimage[dev],geo.dVoxelX*geo.dVoxelY*geo.dVoxelZ/(geo.dDetecU*geo.dDetecV)); + } + + // Now we need to take the image out of the GPU + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipStreamSynchronize(stream[dev*nStreamDevice]); + + num_bytes_img_curr=(size_t)geoArray[img_slice*deviceCount+dev].nVoxelX*(size_t)geoArray[img_slice*deviceCount+dev].nVoxelY*(size_t)geoArray[img_slice*deviceCount+dev].nVoxelZ*sizeof(float); + img_linear_idx_start=(size_t)geo.nVoxelX*(size_t)geo.nVoxelY*(size_t)geoArray[0].nVoxelZ*(size_t)(img_slice*deviceCount+dev); + hipMemcpyAsync(&result[img_linear_idx_start], dimage[dev], num_bytes_img_curr, hipMemcpyDeviceToHost,stream[dev*nStreamDevice+1]); + } + } // end image splits + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDeviceSynchronize(); + } + + + // Clean the GPU + bool two_buffers_used=((((nalpha+split_projections-1)/split_projections)+PROJ_PER_KERNEL-1)/PROJ_PER_KERNEL)>1; + for(unsigned int i=0; i<2;i++){ // 2 buffers (if needed, maybe only 1) + if (!two_buffers_used && i==1) + break; for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipDestroyTextureObject(texProj[i*deviceCount+dev]); + hipFreeArray(d_cuArrTex[i*deviceCount+dev]); + } + } + free(d_cuArrTex); + free(texProj); + + for (dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipFree(dimage[dev]); + } + free(dimage); + + hipFreeHost(projSinCosArray2Host); + hipFreeHost(projParamsArray2Host); + free(partial_projection); + free(proj_split_size); + + freeGeoArray(split_image*deviceCount,geoArray); +#ifndef NO_PINNED_MEMORY + if (isHostRegisterSupported & split_image>1){ + hipHostUnregister(result); + } + if (isHostRegisterSupported){ + hipHostUnregister(projections); + } +#endif + for (int i = 0; i < nStreams; ++i) + hipStreamDestroy(stream[i]); + + hipCheckErrors("hipFree fail"); + +// hipDeviceReset(); // For the Nvidia Visual Profiler + return 0; + +} // END voxel_backprojection + + + + + +void CreateTexture2(const GpuIds& gpuids, float* projectiondata,Geometry geo,hipArray** d_cuArrTex,unsigned int nangles, hipTextureObject_t *texImage,hipStream_t* stream,int nStreamDevice,bool allocate){ + //size_t size_image=geo.nVoxelX*geo.nVoxelY*geo.nVoxelZ; + int num_devices = gpuids.GetLength(); +#if IS_FOR_MATLAB_TIGRE + const hipExtent extent =make_cudaExtent(geo.nDetecV, geo.nDetecU, nangles); +#else + const hipExtent extent =make_cudaExtent(geo.nDetecU, geo.nDetecV, nangles); +#endif + if (allocate){ + for (unsigned int dev = 0; dev < num_devices; dev++){ + hipSetDevice(gpuids[dev]); + + //hipArray Descriptor + hipChannelFormatDesc channelDesc = hipCreateChannelDesc(); + //cuda Array + hipMalloc3DArray(&d_cuArrTex[dev], &channelDesc, extent); + + } + } + for (unsigned int dev = 0; dev < num_devices; dev++){ + hipSetDevice(gpuids[dev]); + hipMemcpy3DParms copyParams = {0}; + //Array creation + copyParams.srcPtr = make_cudaPitchedPtr((void *)projectiondata, extent.width*sizeof(float), extent.width, extent.height); + copyParams.dstArray = d_cuArrTex[dev]; + copyParams.extent = extent; + copyParams.kind = hipMemcpyHostToDevice; + hipMemcpy3DAsync(©Params,stream[dev*nStreamDevice+1]); + } + + //Array creation End + for (unsigned int dev = 0; dev < num_devices; dev++){ + hipSetDevice(gpuids[dev]); + hipResourceDesc texRes; + memset(&texRes, 0, sizeof(hipResourceDesc)); + texRes.resType = hipResourceTypeArray; + texRes.res.array.array = d_cuArrTex[dev]; + hipTextureDesc texDescr; + memset(&texDescr, 0, sizeof(hipTextureDesc)); + texDescr.normalizedCoords = false; + texDescr.filterMode = hipFilterModeLinear; + texDescr.addressMode[0] = hipAddressModeBorder; + texDescr.addressMode[1] = hipAddressModeBorder; + texDescr.addressMode[2] = hipAddressModeBorder; + texDescr.readMode = hipReadModeElementType; + hipCreateTextureObject(&texImage[dev], &texRes, &texDescr, NULL); + } +} +#ifndef BACKPROJECTION_HPP +void splitCTbackprojection(const GpuIds& gpuids, Geometry geo,int nalpha, unsigned int* split_image, unsigned int * split_projections){ + + + // We don't know if the devices are being used. lets check that. and only use the amount of memory we need. + + size_t mem_GPU_global; + checkFreeMemory(gpuids, &mem_GPU_global); + const int deviceCount = gpuids.GetLength(); + + // Compute how much memory each of the relevant memory pieces need + size_t mem_image= (unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY*(unsigned long long)geo.nVoxelZ*sizeof(float); + size_t mem_proj= (unsigned long long)geo.nDetecU*(unsigned long long)geo.nDetecV*sizeof(float); + + + + + // Does everything fit in the GPU? + + if(mem_image/deviceCount+mem_proj*PROJ_PER_KERNEL*2x=Px.x-P.x; deltaX->y=Px.y-P.y; deltaX->z=Px.z-P.z; + deltaY->x=Py.x-P.x; deltaY->y=Py.y-P.y; deltaY->z=Py.z-P.z; + deltaZ->x=Pz.x-P.x; deltaZ->y=Pz.y-P.y; deltaZ->z=Pz.z-P.z; + + + *xyzorigin=P.to_float(); + *S=source.to_float(); +} // END computeDeltasCube + +void checkFreeMemory(const GpuIds& gpuids,size_t *mem_GPU_global){ + size_t memfree; + size_t memtotal; + const int gpuids.GetLength(); + + for (int dev = 0; dev < deviceCount; dev++){ + hipSetDevice(gpuids[dev]); + hipMemGetInfo(&memfree,&memtotal); + if(dev==0) *mem_GPU_global=memfree; + if(memfree -#include -#include -#include "voxel_backprojection.hpp" -#include "voxel_backprojection_parallel.hpp" - -#include "TIGRE_common.hpp" -#include - -// https://stackoverflow.com/questions/16282136/is-there-a-cuda-equivalent-of-perror -#define cudaCheckErrors(msg) \ -do { \ - cudaError_t __err = cudaGetLastError(); \ - if (__err != cudaSuccess) { \ - mexPrintf("%s \n",msg);\ - mexErrMsgIdAndTxt("CBCT:CUDA:Atb",cudaGetErrorString(__err));\ - } \ -} while (0) - - -#define MAXTREADS 1024 - /*GEOMETRY DEFINITION - * - * Detector plane, behind - * |-----------------------------| - * | | - * | | - * | | - * | | - * | +--------+ | - * | / /| | - * A Z | / / |*D | - * | | +--------+ | | - * | | | | | | - * | | | *O | + | - * *--->y | | | / | - * / | | |/ | - * V X | +--------+ | - * |-----------------------------| - * - * *S - * - * - * - * - * - **/ -void CreateTextureParallel( float* projectiondata,Geometry geo,cudaArray** d_cuArrTex,unsigned int nangles, cudaTextureObject_t *texImage,cudaStream_t* stream, bool allocate); - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// RB, 10/31/2016: Add constant memory arrays to store parameters for all projections to be analyzed during a single kernel call -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -// The optimal values of two constants obtained by RB on NVIDIA Quadro K2200 (4 GB RAM, 640 CUDA cores) for 512^3 volume and 512^3 projections (512 proj, each 512 x 512) were: -// PROJ_PER_KERNEL = 32 or 16 (very similar times) -// VOXELS_PER_THREAD = 8 -// Speedup of the entire FDK backprojection (not only kernel run, also memcpy etc.) was nearly 4x relative to the original (single projection, single voxel per thread) code. -// (e.g. 16.2 s vs. ~62 s). - -const int PROJ_PER_KERNEL = 32; // Number of 2D projections to be analyzed by a single thread. This can be tweaked to see what works best. 32 was the optimal value in the paper by Zinsser and Keck. -const int VOXELS_PER_THREAD = 8; // Number of voxels to be computed by s single thread. Can be tweaked to see what works best. 4 was the optimal value in the paper by Zinsser and Keck. - -// We have PROJ_PER_KERNEL projections and we need 6 parameters for each projection: -// deltaX, deltaY, deltaZ, xyzOrigin, offOrig, offDetec -// So we need to keep PROJ_PER_KERNEL*6 values in our deltas array FOR EACH CALL to our main kernel -// (they will be updated in the main loop before each kernel call). - -__constant__ Point3D projParamsArrayDevParallel[6*PROJ_PER_KERNEL]; // Dev means it is on device - -// We also need a corresponding array on the host side to be filled before each kernel call, then copied to the device (array in constant memory above) -// Point3D projParamsArrayHostParallel[6*PROJ_PER_KERNEL]; // Host means it is host memory - -// Now we also need to store sinAlpha and cosAlpha for each projection (two floats per projection) -__constant__ float projSinCosArrayDevParallel[3*PROJ_PER_KERNEL]; - -// float projSinCosArrayHostParallel[3*PROJ_PER_KERNEL]; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// END RB, 10/31/2016: Add constant memory arrays to store parameters for all projections to be analyzed during a single kernel call -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - - -//______________________________________________________________________________ -// -// Function: kernelPixelBackprojectionFDK -// -// Description: Main FDK backprojection kernel -//______________________________________________________________________________ - -__global__ void kernelPixelBackprojection_parallel(const Geometry geo, float* image,const int currProjSetNumber, const int totalNoOfProjections,cudaTextureObject_t tex) -{ - - // Old kernel call signature: - // kernelPixelBackprojectionFDK<<>>(geo,dimage,i,deltaX,deltaY,deltaZ,xyzOrigin,offOrig,offDetec,sinalpha,cosalpha); - // We just read in most of the params from the constant memory instead of getting them from the param list. - // This is because we now have MANY params, since single kernel processes more than one projection! - /* __global__ void kernelPixelBackprojectionFDK(const Geometry geo, - * float* image, - * const int indAlpha, - * const Point3D deltaX , - * const Point3D deltaY, - * const Point3D deltaZ, - * const Point3D xyzOrigin, - * const Point3D xyzOffset, - * const Point3D uv0Offset, - * const float sinalpha, - * const float cosalpha){ - */ - unsigned long long indY = blockIdx.y * blockDim.y + threadIdx.y; - unsigned long long indX = blockIdx.x * blockDim.x + threadIdx.x; - // unsigned long startIndZ = blockIdx.z * blockDim.z + threadIdx.z; // This is only STARTING z index of the column of voxels that the thread will handle - unsigned long long startIndZ = blockIdx.z * VOXELS_PER_THREAD + threadIdx.z; // This is only STARTING z index of the column of voxels that the thread will handle - //Make sure we don't go out of bounds - if (indX>=geo.nVoxelX || indY>=geo.nVoxelY || startIndZ>=geo.nVoxelZ) - return; - - // We'll keep a local auxiliary array of values of a column of voxels that this thread will update - float voxelColumn[VOXELS_PER_THREAD]; - - // First we need to copy the curent 3D volume values from the column to our auxiliary array so that we can then - // work on them (update them by computing values from multiple projections) locally - avoiding main memory reads/writes - - unsigned long colIdx; - - for(colIdx=0; colIdx=geo.nVoxelZ) - break; // break the loop. - - unsigned long long idx =indZ*(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY+indY*(unsigned long long)geo.nVoxelX + indX; - voxelColumn[colIdx] = image[idx]; // Read the current volume value that we'll update by computing values from MULTIPLE projections (not just one) - // We'll be updating the local (register) variable, avoiding reads/writes from the slow main memory. - } // END copy 3D volume voxels to local array - - // Now iterate through projections - for(unsigned long projNumber=0; projNumber=totalNoOfProjections) - break; - - Point3D deltaX = projParamsArrayDevParallel[6*projNumber]; // 6*projNumber because we have 6 Point3D values per projection - Point3D deltaY = projParamsArrayDevParallel[6*projNumber+1]; - Point3D deltaZ = projParamsArrayDevParallel[6*projNumber+2]; - Point3D xyzOrigin = projParamsArrayDevParallel[6*projNumber+3]; - Point3D xyzOffset = projParamsArrayDevParallel[6*projNumber+4]; - Point3D S = projParamsArrayDevParallel[6*projNumber+5]; - - float DSD = projSinCosArrayDevParallel[3*projNumber]; // 2*projNumber because we have 2 float (sin or cos angle) values per projection - float DSO = projSinCosArrayDevParallel[3*projNumber+1]; - float COR = projSinCosArrayDevParallel[3*projNumber+2]; - - // Geometric trasnformations: - //Source, scaled XYZ coordinates - - // Now iterate through Z in our voxel column FOR A GIVEN PROJECTION - for(colIdx=0; colIdx=geo.nVoxelZ) - break; // break the loop. - - // "XYZ" in the scaled coordinate system of the current point. The image is rotated with the projection angles. - Point3D P; - S.x=DSO; - P.x=(xyzOrigin.x+indX*deltaX.x+indY*deltaY.x+indZ*deltaZ.x); - P.y=(xyzOrigin.y+indX*deltaX.y+indY*deltaY.y+indZ*deltaZ.y)-COR/geo.dDetecU; - P.z=(xyzOrigin.z+indX*deltaX.z+indY*deltaY.z+indZ*deltaZ.z); - S.y=P.y;S.z=P.z; - - // This is the vector defining the line from the source to the Voxel - float vectX,vectY,vectZ; - vectX=(P.x -S.x); - vectY=(P.y -S.y); - vectZ=(P.z -S.z); - - // Get the coordinates in the detector UV where the mid point of the voxel is projected. - float t=(DSO-DSD /*-DOD*/ - S.x)/vectX; - float y,z; - y=vectY*t+S.y; - z=vectZ*t+S.z; - float u,v; - u=y+geo.nDetecU/2.0f-0.5f; - v=z+geo.nDetecV/2.0f-0.5f; - - - - // Get Value in the computed (U,V) and multiply by the corresponding weight. - // indAlpha is the ABSOLUTE number of projection in the projection array (NOT the current number of projection set!) -#if IS_FOR_MATLAB_TIGRE - voxelColumn[colIdx]+=tex3D(tex, v+0.5f, u+0.5f ,indAlpha+0.5f); -#else - voxelColumn[colIdx]+=tex3D(tex, u+0.5f, v+0.5f ,indAlpha+0.5f); -#endif - - } // END iterating through column of voxels - - } // END iterating through multiple projections - - // And finally copy the updated local voxelColumn array back to our 3D volume (main memory) - for(colIdx=0; colIdx=geo.nVoxelZ) - break; // break the loop. - - unsigned long long idx =indZ*(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY+indY*(unsigned long long)geo.nVoxelX + indX; - image[idx] = voxelColumn[colIdx]; // Read the current volume value that we'll update by computing values from MULTIPLE projections (not just one) - // We'll be updating the local (register) variable, avoiding reads/writes from the slow main memory. - // According to references (Papenhausen), doing = is better than +=, since += requires main memory read followed by a write. - // We did all the reads into the local array at the BEGINNING of this kernel. According to Papenhausen, this type of read-write split is - // better for avoiding memory congestion. - } // END copy updated voxels from local array to our 3D volume - -} // END kernelPixelBackprojectionFDK - - - - -//______________________________________________________________________________ -// -// Function: voxel_backprojection_parallel -// -// Description: Main host function for FDK backprojection (invokes the kernel) -//______________________________________________________________________________ - -int voxel_backprojection_parallel(float * projections, Geometry geo, float* result,float const * const alphas, int nalpha, const GpuIds& gpuids) -{ - if (gpuids.GetLength() == 0) { - cudaSetDevice(0); - } else { - cudaSetDevice(gpuids[0]); - } - - /* - * Allocate texture memory on the device - */ - // copy data to CUDA memory - //If it is the first time, lets make sure our image is zeroed. - int nStreamDevice=2; - int nStreams=nStreamDevice; - cudaStream_t* stream=(cudaStream_t*)malloc(nStreams*sizeof(cudaStream_t));; - - for (int i = 0; i < nStreamDevice; ++i){ - cudaStreamCreate(&stream[i]); - - - } - //Pagelock memory for synchronous copy. - // Lets try to make the host memory pinned: - // We laredy queried the GPU and assuemd they are the same, thus should have the same attributes. - int isHostRegisterSupported = 0; -#if CUDART_VERSION >= 9020 - cudaDeviceGetAttribute(&isHostRegisterSupported,cudaDevAttrHostRegisterSupported,gpuids[0]); -#endif - if (isHostRegisterSupported){ - cudaHostRegister(projections, (size_t)geo.nDetecU*(size_t)geo.nDetecV*(size_t)nalpha*(size_t)sizeof(float),cudaHostRegisterPortable); - } - cudaCheckErrors("Error pinning memory"); - - - // Allocate result image memory - size_t num_bytes = geo.nVoxelX*geo.nVoxelY*geo.nVoxelZ * sizeof(float); - float* dimage; - cudaMalloc((void**)&dimage, num_bytes); - cudaMemset(dimage,0,num_bytes); - cudaCheckErrors("cudaMalloc fail"); - - - Point3D* projParamsArrayHostParallel; - cudaMallocHost((void**)&projParamsArrayHostParallel,6*PROJ_PER_KERNEL*sizeof(Point3D)); - float* projSinCosArrayHostParallel; - cudaMallocHost((void**)&projSinCosArrayHostParallel,3*PROJ_PER_KERNEL*sizeof(float)); - - - // Texture buffer objects - cudaTextureObject_t *texProj; - cudaArray **d_cuArrTex; - texProj =(cudaTextureObject_t*)malloc(2*sizeof(cudaTextureObject_t)); - d_cuArrTex =(cudaArray**)malloc(2*sizeof(cudaArray*)); - - - - unsigned int proj_split_overlap_number; - unsigned int split_projections=1; - // Start with the main loop. The Projection data needs to be allocated and dealocated in the main loop - // as due to the nature of cudaArrays, we can not reuse them. This should not be a problem for the fast execution - // of the code, as repeated allocation and deallocation only happens when the projection data is very very big, - // and therefore allcoation time should be negligible, fluctuation of other computations should mask the time. - unsigned long long proj_linear_idx_start; - unsigned int current_proj_split_size,current_proj_overlap_split_size; - size_t num_bytes_img_curr; - size_t img_linear_idx_start; - - - current_proj_split_size=nalpha; - // We are going to split it in the same amount of kernels we need to execute. - proj_split_overlap_number=(current_proj_split_size+PROJ_PER_KERNEL-1)/PROJ_PER_KERNEL; - - - // Create pointer to pointers of projections and precompute their location and size. - - float ** partial_projection=(float**)malloc(current_proj_split_size*sizeof(float*)); - size_t * proj_split_size=(size_t*)malloc(current_proj_split_size*sizeof(size_t*)); - - for(unsigned int proj_block_split=0; proj_block_split=proj_split_size[proj_block_split]) - break; // Exit the loop. Even when we leave the param arrays only partially filled, this is OK, since the kernel will check bounds anyway. - - if(currProjNumber_global>=nalpha) - break; // Exit the loop. Even when we leave the param arrays only partially filled, this is OK, since the kernel will check bounds anyway. - - Point3D deltaX,deltaY,deltaZ,xyzOrigin, offOrig, /*offDetec,*/source; - float sinalpha,cosalpha; - - geo.alpha=-alphas[currProjNumber_global*3]; - geo.theta=-alphas[currProjNumber_global*3+1]; - geo.psi =-alphas[currProjNumber_global*3+2]; - - //sinalpha=sin(geo.alpha); -// cosalpha=cos(geo.alpha); - - projSinCosArrayHostParallel[3*j]=geo.DSD[currProjNumber_global]; // 3*j because we have 3 float (sin or cos angle) values per projection - projSinCosArrayHostParallel[3*j+1]=geo.DSO[currProjNumber_global]; - projSinCosArrayHostParallel[3*j+2]=geo.COR[currProjNumber_global]; - - //computeDeltasCubeParallel(geo,geo.alpha,currProjNumber,&xyzOrigin,&deltaX,&deltaY,&deltaZ,&source); - computeDeltasCubeParallel(geo,currProjNumber_global,&xyzOrigin,&deltaX,&deltaY,&deltaZ,&source); - - offOrig.x=geo.offOrigX[currProjNumber_global]; - offOrig.y=geo.offOrigY[currProjNumber_global]; - - - projParamsArrayHostParallel[6*j]=deltaX; // 6*j because we have 6 Point3D values per projection - projParamsArrayHostParallel[6*j+1]=deltaY; - projParamsArrayHostParallel[6*j+2]=deltaZ; - projParamsArrayHostParallel[6*j+3]=xyzOrigin; - projParamsArrayHostParallel[6*j+4]=offOrig; - projParamsArrayHostParallel[6*j+5]=source; - } // END for (preparing params for kernel call) - - // Copy the prepared parameter arrays to constant memory to make it available for the kernel - - cudaMemcpyToSymbolAsync(projSinCosArrayDevParallel, projSinCosArrayHostParallel, sizeof(float)*3*PROJ_PER_KERNEL,0,cudaMemcpyHostToDevice,stream[0]); - cudaMemcpyToSymbolAsync(projParamsArrayDevParallel, projParamsArrayHostParallel, sizeof(Point3D)*6*PROJ_PER_KERNEL,0,cudaMemcpyHostToDevice,stream[0]); - cudaStreamSynchronize(stream[0]); - - kernelPixelBackprojection_parallel<<>>(geo,dimage,i,proj_split_size[proj_block_split],texProj[(proj_block_split%2)]); - } // END for - - ////////////////////////////////////////////////////////////////////////////////////// - // END Main reconstruction loop: go through projections (rotation angles) and backproject - ////////////////////////////////////////////////////////////////////////////////////// - } - cudaDeviceSynchronize(); - cudaMemcpy(result, dimage, num_bytes, cudaMemcpyDeviceToHost); - cudaCheckErrors("cudaMemcpy result fail"); - - free(partial_projection); - free(proj_split_size); - - bool two_buffers_used=((((nalpha+split_projections-1)/split_projections)+PROJ_PER_KERNEL-1)/PROJ_PER_KERNEL)>1; - for(unsigned int i=0; i<2;i++){ // 2 buffers (if needed, maybe only 1) - if (!two_buffers_used && i==1) - break; - cudaDestroyTextureObject(texProj[i]); - cudaFreeArray(d_cuArrTex[i]); - } - free(texProj); - - free(d_cuArrTex); - cudaFreeHost(projSinCosArrayHostParallel); - cudaFreeHost(projParamsArrayHostParallel); - - cudaFree(dimage); - if (isHostRegisterSupported){ - cudaHostUnregister(projections); - } - for (int i = 0; i < nStreams; ++i) - cudaStreamDestroy(stream[i]); - -// cudaDeviceReset(); - return 0; - -} // END voxel_backprojection - -void computeDeltasCubeParallel(Geometry geo, int i, Point3D* xyzorigin, Point3D* deltaX, Point3D* deltaY, Point3D* deltaZ,Point3D *S) -{ - - Point3Ddouble P, Px,Py,Pz; - // Get coords of Img(0,0,0) - P.x=-(geo.sVoxelX/2-geo.dVoxelX/2)+geo.offOrigX[i]; - P.y=-(geo.sVoxelY/2-geo.dVoxelY/2)+geo.offOrigY[i]; - P.z=-(geo.sVoxelZ/2-geo.dVoxelZ/2)+geo.offOrigZ[i]; - - // Get coors from next voxel in each direction - Px.x=P.x+geo.dVoxelX; Py.x=P.x; Pz.x=P.x; - Px.y=P.y; Py.y=P.y+geo.dVoxelY; Pz.y=P.y; - Px.z=P.z; Py.z=P.z; Pz.z=P.z+geo.dVoxelZ; - - - - // Rotate image around X axis (this is equivalent of rotating the source and detector) RZ RY RZ - eulerZYZT(geo,&P); - eulerZYZT(geo,&Px); - eulerZYZT(geo,&Py); - eulerZYZT(geo,&Pz); - - //detector offset - P.z =P.z-geo.offDetecV[i]; P.y =P.y-geo.offDetecU[i]; - Px.z =Px.z-geo.offDetecV[i]; Px.y =Px.y-geo.offDetecU[i]; - Py.z =Py.z-geo.offDetecV[i]; Py.y =Py.y-geo.offDetecU[i]; - Pz.z =Pz.z-geo.offDetecV[i]; Pz.y =Pz.y-geo.offDetecU[i]; - - //Detector Roll pitch Yaw - // - // - // first, we need to offset everything so (0,0,0) is the center of the detector - // Only X is required for that - P.x=P.x+(geo.DSD[i]-geo.DSO[i]); - Px.x=Px.x+(geo.DSD[i]-geo.DSO[i]); - Py.x=Py.x+(geo.DSD[i]-geo.DSO[i]); - Pz.x=Pz.x+(geo.DSD[i]-geo.DSO[i]); - - rollPitchYawT(geo,i,&P); - rollPitchYawT(geo,i,&Px); - rollPitchYawT(geo,i,&Py); - rollPitchYawT(geo,i,&Pz); - - P.x=P.x-(geo.DSD[i]-geo.DSO[i]); - Px.x=Px.x-(geo.DSD[i]-geo.DSO[i]); - Py.x=Py.x-(geo.DSD[i]-geo.DSO[i]); - Pz.x=Pz.x-(geo.DSD[i]-geo.DSO[i]); - - - Point3Ddouble source; - source.x=0; - source.y=-geo.offDetecU[i]; - source.z=-geo.offDetecV[i]; - - rollPitchYawT(geo,i,&source); - source.x=source.x-(geo.DSD[i]-geo.DSO[i]); - - P.z =P.z /geo.dDetecV; P.y =P.y/geo.dDetecU; - Px.z=Px.z/geo.dDetecV; Px.y=Px.y/geo.dDetecU; - Py.z=Py.z/geo.dDetecV; Py.y=Py.y/geo.dDetecU; - Pz.z=Pz.z/geo.dDetecV; Pz.y=Pz.y/geo.dDetecU; - - source.z=source.z/geo.dDetecV; source.y=source.y/geo.dDetecU; - - // get deltas of the changes in voxels - deltaX->x=Px.x-P.x; deltaX->y=Px.y-P.y; deltaX->z=Px.z-P.z; - deltaY->x=Py.x-P.x; deltaY->y=Py.y-P.y; deltaY->z=Py.z-P.z; - deltaZ->x=Pz.x-P.x; deltaZ->y=Pz.y-P.y; deltaZ->z=Pz.z-P.z; - - - // cast the results from the double precision calculations back to float - *xyzorigin=P.to_float(); - *S=source.to_float(); - - -} // END computeDeltasCube -void CreateTextureParallel(float* projectiondata,Geometry geo,cudaArray** d_cuArrTex,unsigned int nangles, cudaTextureObject_t *texImage,cudaStream_t* stream, bool alloc) -{ - //cudaArray Descriptor -#if IS_FOR_MATLAB_TIGRE - const cudaExtent extent =make_cudaExtent(geo.nDetecV, geo.nDetecU, nangles); -#else - const cudaExtent extent =make_cudaExtent(geo.nDetecU, geo.nDetecV, nangles); -#endif - cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(); - //cuda Array - if (alloc){ - cudaMalloc3DArray(&d_cuArrTex[0], &channelDesc, extent); - cudaCheckErrors("Texture memory allocation fail"); - } - cudaMemcpy3DParms copyParams = {0}; - - - //Array creation - copyParams.srcPtr = make_cudaPitchedPtr((void *)projectiondata, extent.width*sizeof(float), extent.width, extent.height); - copyParams.dstArray = d_cuArrTex[0]; - copyParams.extent = extent; - copyParams.kind = cudaMemcpyHostToDevice; - cudaMemcpy3DAsync(©Params,stream[0+1]); - cudaCheckErrors("Texture memory data copy fail"); - //Array creation End - - cudaResourceDesc texRes; - memset(&texRes, 0, sizeof(cudaResourceDesc)); - texRes.resType = cudaResourceTypeArray; - texRes.res.array.array = d_cuArrTex[0]; - cudaTextureDesc texDescr; - memset(&texDescr, 0, sizeof(cudaTextureDesc)); - texDescr.normalizedCoords = false; - texDescr.filterMode = cudaFilterModeLinear; - texDescr.addressMode[0] = cudaAddressModeBorder; - texDescr.addressMode[1] = cudaAddressModeBorder; - texDescr.addressMode[2] = cudaAddressModeBorder; - texDescr.readMode = cudaReadModeElementType; - cudaCreateTextureObject(&texImage[0], &texRes, &texDescr, NULL); - cudaCheckErrors("Texture object creation fail"); - +/*------------------------------------------------------------------------- + * + * CUDA function for backrpojection for parallel beam + * + * + * CODE by Ander Biguri + * Optimized and modified by RB + * --------------------------------------------------------------------------- + * --------------------------------------------------------------------------- + * Copyright (c) 2015, University of Bath and CERN- European Organization for + * Nuclear Research + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * --------------------------------------------------------------------------- + * + * Contact: tigre.toolbox@gmail.com + * Codes : https://github.com/CERN/TIGRE + * --------------------------------------------------------------------------- + */ + + +#define PI_2 1.57079632679489661923 +#include +#include +#include +#include "voxel_backprojection.hpp" +#include "voxel_backprojection_parallel.hpp" + +#include "TIGRE_common.hpp" +#include + +// https://stackoverflow.com/questions/16282136/is-there-a-cuda-equivalent-of-perror +#define hipCheckErrors(msg) \ +do { \ + hipError_t __err = hipGetLastError(); \ + if (__err != hipSuccess) { \ + mexPrintf("%s \n",msg);\ + mexErrMsgIdAndTxt("CBCT:CUDA:Atb",hipGetErrorString(__err));\ + } \ +} while (0) + + +#define MAXTREADS 1024 + /*GEOMETRY DEFINITION + * + * Detector plane, behind + * |-----------------------------| + * | | + * | | + * | | + * | | + * | +--------+ | + * | / /| | + * A Z | / / |*D | + * | | +--------+ | | + * | | | | | | + * | | | *O | + | + * *--->y | | | / | + * / | | |/ | + * V X | +--------+ | + * |-----------------------------| + * + * *S + * + * + * + * + * + **/ +void CreateTextureParallel( float* projectiondata,Geometry geo,hipArray** d_cuArrTex,unsigned int nangles, hipTextureObject_t *texImage,hipStream_t* stream, bool allocate); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// RB, 10/31/2016: Add constant memory arrays to store parameters for all projections to be analyzed during a single kernel call +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// The optimal values of two constants obtained by RB on NVIDIA Quadro K2200 (4 GB RAM, 640 CUDA cores) for 512^3 volume and 512^3 projections (512 proj, each 512 x 512) were: +// PROJ_PER_KERNEL = 32 or 16 (very similar times) +// VOXELS_PER_THREAD = 8 +// Speedup of the entire FDK backprojection (not only kernel run, also memcpy etc.) was nearly 4x relative to the original (single projection, single voxel per thread) code. +// (e.g. 16.2 s vs. ~62 s). + +const int PROJ_PER_KERNEL = 32; // Number of 2D projections to be analyzed by a single thread. This can be tweaked to see what works best. 32 was the optimal value in the paper by Zinsser and Keck. +const int VOXELS_PER_THREAD = 8; // Number of voxels to be computed by s single thread. Can be tweaked to see what works best. 4 was the optimal value in the paper by Zinsser and Keck. + +// We have PROJ_PER_KERNEL projections and we need 6 parameters for each projection: +// deltaX, deltaY, deltaZ, xyzOrigin, offOrig, offDetec +// So we need to keep PROJ_PER_KERNEL*6 values in our deltas array FOR EACH CALL to our main kernel +// (they will be updated in the main loop before each kernel call). + +__constant__ Point3D projParamsArrayDevParallel[6*PROJ_PER_KERNEL]; // Dev means it is on device + +// We also need a corresponding array on the host side to be filled before each kernel call, then copied to the device (array in constant memory above) +// Point3D projParamsArrayHostParallel[6*PROJ_PER_KERNEL]; // Host means it is host memory + +// Now we also need to store sinAlpha and cosAlpha for each projection (two floats per projection) +__constant__ float projSinCosArrayDevParallel[3*PROJ_PER_KERNEL]; + +// float projSinCosArrayHostParallel[3*PROJ_PER_KERNEL]; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// END RB, 10/31/2016: Add constant memory arrays to store parameters for all projections to be analyzed during a single kernel call +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + + +//______________________________________________________________________________ +// +// Function: kernelPixelBackprojectionFDK +// +// Description: Main FDK backprojection kernel +//______________________________________________________________________________ + +__global__ void kernelPixelBackprojection_parallel(const Geometry geo, float* image,const int currProjSetNumber, const int totalNoOfProjections,hipTextureObject_t tex) +{ + + // Old kernel call signature: + // kernelPixelBackprojectionFDK<<>>(geo,dimage,i,deltaX,deltaY,deltaZ,xyzOrigin,offOrig,offDetec,sinalpha,cosalpha); + // We just read in most of the params from the constant memory instead of getting them from the param list. + // This is because we now have MANY params, since single kernel processes more than one projection! + /* __global__ void kernelPixelBackprojectionFDK(const Geometry geo, + * float* image, + * const int indAlpha, + * const Point3D deltaX , + * const Point3D deltaY, + * const Point3D deltaZ, + * const Point3D xyzOrigin, + * const Point3D xyzOffset, + * const Point3D uv0Offset, + * const float sinalpha, + * const float cosalpha){ + */ + unsigned long long indY = blockIdx.y * blockDim.y + threadIdx.y; + unsigned long long indX = blockIdx.x * blockDim.x + threadIdx.x; + // unsigned long startIndZ = blockIdx.z * blockDim.z + threadIdx.z; // This is only STARTING z index of the column of voxels that the thread will handle + unsigned long long startIndZ = blockIdx.z * VOXELS_PER_THREAD + threadIdx.z; // This is only STARTING z index of the column of voxels that the thread will handle + //Make sure we don't go out of bounds + if (indX>=geo.nVoxelX || indY>=geo.nVoxelY || startIndZ>=geo.nVoxelZ) + return; + + // We'll keep a local auxiliary array of values of a column of voxels that this thread will update + float voxelColumn[VOXELS_PER_THREAD]; + + // First we need to copy the curent 3D volume values from the column to our auxiliary array so that we can then + // work on them (update them by computing values from multiple projections) locally - avoiding main memory reads/writes + + unsigned long colIdx; + + for(colIdx=0; colIdx=geo.nVoxelZ) + break; // break the loop. + + unsigned long long idx =indZ*(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY+indY*(unsigned long long)geo.nVoxelX + indX; + voxelColumn[colIdx] = image[idx]; // Read the current volume value that we'll update by computing values from MULTIPLE projections (not just one) + // We'll be updating the local (register) variable, avoiding reads/writes from the slow main memory. + } // END copy 3D volume voxels to local array + + // Now iterate through projections + for(unsigned long projNumber=0; projNumber=totalNoOfProjections) + break; + + Point3D deltaX = projParamsArrayDevParallel[6*projNumber]; // 6*projNumber because we have 6 Point3D values per projection + Point3D deltaY = projParamsArrayDevParallel[6*projNumber+1]; + Point3D deltaZ = projParamsArrayDevParallel[6*projNumber+2]; + Point3D xyzOrigin = projParamsArrayDevParallel[6*projNumber+3]; + Point3D xyzOffset = projParamsArrayDevParallel[6*projNumber+4]; + Point3D S = projParamsArrayDevParallel[6*projNumber+5]; + + float DSD = projSinCosArrayDevParallel[3*projNumber]; // 2*projNumber because we have 2 float (sin or cos angle) values per projection + float DSO = projSinCosArrayDevParallel[3*projNumber+1]; + float COR = projSinCosArrayDevParallel[3*projNumber+2]; + + // Geometric trasnformations: + //Source, scaled XYZ coordinates + + // Now iterate through Z in our voxel column FOR A GIVEN PROJECTION + for(colIdx=0; colIdx=geo.nVoxelZ) + break; // break the loop. + + // "XYZ" in the scaled coordinate system of the current point. The image is rotated with the projection angles. + Point3D P; + S.x=DSO; + P.x=(xyzOrigin.x+indX*deltaX.x+indY*deltaY.x+indZ*deltaZ.x); + P.y=(xyzOrigin.y+indX*deltaX.y+indY*deltaY.y+indZ*deltaZ.y)-COR/geo.dDetecU; + P.z=(xyzOrigin.z+indX*deltaX.z+indY*deltaY.z+indZ*deltaZ.z); + S.y=P.y;S.z=P.z; + + // This is the vector defining the line from the source to the Voxel + float vectX,vectY,vectZ; + vectX=(P.x -S.x); + vectY=(P.y -S.y); + vectZ=(P.z -S.z); + + // Get the coordinates in the detector UV where the mid point of the voxel is projected. + float t=(DSO-DSD /*-DOD*/ - S.x)/vectX; + float y,z; + y=vectY*t+S.y; + z=vectZ*t+S.z; + float u,v; + u=y+geo.nDetecU/2.0f-0.5f; + v=z+geo.nDetecV/2.0f-0.5f; + + + + // Get Value in the computed (U,V) and multiply by the corresponding weight. + // indAlpha is the ABSOLUTE number of projection in the projection array (NOT the current number of projection set!) +#if IS_FOR_MATLAB_TIGRE + voxelColumn[colIdx]+=tex3D(tex, v+0.5f, u+0.5f ,indAlpha+0.5f); +#else + voxelColumn[colIdx]+=tex3D(tex, u+0.5f, v+0.5f ,indAlpha+0.5f); +#endif + + } // END iterating through column of voxels + + } // END iterating through multiple projections + + // And finally copy the updated local voxelColumn array back to our 3D volume (main memory) + for(colIdx=0; colIdx=geo.nVoxelZ) + break; // break the loop. + + unsigned long long idx =indZ*(unsigned long long)geo.nVoxelX*(unsigned long long)geo.nVoxelY+indY*(unsigned long long)geo.nVoxelX + indX; + image[idx] = voxelColumn[colIdx]; // Read the current volume value that we'll update by computing values from MULTIPLE projections (not just one) + // We'll be updating the local (register) variable, avoiding reads/writes from the slow main memory. + // According to references (Papenhausen), doing = is better than +=, since += requires main memory read followed by a write. + // We did all the reads into the local array at the BEGINNING of this kernel. According to Papenhausen, this type of read-write split is + // better for avoiding memory congestion. + } // END copy updated voxels from local array to our 3D volume + +} // END kernelPixelBackprojectionFDK + + + + +//______________________________________________________________________________ +// +// Function: voxel_backprojection_parallel +// +// Description: Main host function for FDK backprojection (invokes the kernel) +//______________________________________________________________________________ + +int voxel_backprojection_parallel(float * projections, Geometry geo, float* result,float const * const alphas, int nalpha, const GpuIds& gpuids) +{ + if (gpuids.GetLength() == 0) { + hipSetDevice(0); + } else { + hipSetDevice(gpuids[0]); + } + + /* + * Allocate texture memory on the device + */ + // copy data to CUDA memory + //If it is the first time, lets make sure our image is zeroed. + int nStreamDevice=2; + int nStreams=nStreamDevice; + hipStream_t* stream=(hipStream_t*)malloc(nStreams*sizeof(hipStream_t));; + + for (int i = 0; i < nStreamDevice; ++i){ + hipStreamCreate(&stream[i]); + + + } + //Pagelock memory for synchronous copy. + // Lets try to make the host memory pinned: + // We laredy queried the GPU and assuemd they are the same, thus should have the same attributes. + int isHostRegisterSupported = 0; +#if CUDART_VERSION >= 9020 + hipDeviceGetAttribute(&isHostRegisterSupported,hipDevAttrHostRegisterSupported,gpuids[0]); +#endif + if (isHostRegisterSupported){ + hipHostRegister(projections, (size_t)geo.nDetecU*(size_t)geo.nDetecV*(size_t)nalpha*(size_t)sizeof(float),hipHostRegisterPortable); + } + hipCheckErrors("Error pinning memory"); + + + // Allocate result image memory + size_t num_bytes = geo.nVoxelX*geo.nVoxelY*geo.nVoxelZ * sizeof(float); + float* dimage; + hipMalloc((void**)&dimage, num_bytes); + hipMemset(dimage,0,num_bytes); + hipCheckErrors("hipMalloc fail"); + + + Point3D* projParamsArrayHostParallel; + hipMallocHost((void**)&projParamsArrayHostParallel,6*PROJ_PER_KERNEL*sizeof(Point3D)); + float* projSinCosArrayHostParallel; + hipMallocHost((void**)&projSinCosArrayHostParallel,3*PROJ_PER_KERNEL*sizeof(float)); + + + // Texture buffer objects + hipTextureObject_t *texProj; + hipArray **d_cuArrTex; + texProj =(hipTextureObject_t*)malloc(2*sizeof(hipTextureObject_t)); + d_cuArrTex =(hipArray**)malloc(2*sizeof(hipArray*)); + + + + unsigned int proj_split_overlap_number; + unsigned int split_projections=1; + // Start with the main loop. The Projection data needs to be allocated and dealocated in the main loop + // as due to the nature of hipArrays, we can not reuse them. This should not be a problem for the fast execution + // of the code, as repeated allocation and deallocation only happens when the projection data is very very big, + // and therefore allcoation time should be negligible, fluctuation of other computations should mask the time. + unsigned long long proj_linear_idx_start; + unsigned int current_proj_split_size,current_proj_overlap_split_size; + size_t num_bytes_img_curr; + size_t img_linear_idx_start; + + + current_proj_split_size=nalpha; + // We are going to split it in the same amount of kernels we need to execute. + proj_split_overlap_number=(current_proj_split_size+PROJ_PER_KERNEL-1)/PROJ_PER_KERNEL; + + + // Create pointer to pointers of projections and precompute their location and size. + + float ** partial_projection=(float**)malloc(current_proj_split_size*sizeof(float*)); + size_t * proj_split_size=(size_t*)malloc(current_proj_split_size*sizeof(size_t*)); + + for(unsigned int proj_block_split=0; proj_block_split=proj_split_size[proj_block_split]) + break; // Exit the loop. Even when we leave the param arrays only partially filled, this is OK, since the kernel will check bounds anyway. + + if(currProjNumber_global>=nalpha) + break; // Exit the loop. Even when we leave the param arrays only partially filled, this is OK, since the kernel will check bounds anyway. + + Point3D deltaX,deltaY,deltaZ,xyzOrigin, offOrig, /*offDetec,*/source; + float sinalpha,cosalpha; + + geo.alpha=-alphas[currProjNumber_global*3]; + geo.theta=-alphas[currProjNumber_global*3+1]; + geo.psi =-alphas[currProjNumber_global*3+2]; + + //sinalpha=sin(geo.alpha); +// cosalpha=cos(geo.alpha); + + projSinCosArrayHostParallel[3*j]=geo.DSD[currProjNumber_global]; // 3*j because we have 3 float (sin or cos angle) values per projection + projSinCosArrayHostParallel[3*j+1]=geo.DSO[currProjNumber_global]; + projSinCosArrayHostParallel[3*j+2]=geo.COR[currProjNumber_global]; + + //computeDeltasCubeParallel(geo,geo.alpha,currProjNumber,&xyzOrigin,&deltaX,&deltaY,&deltaZ,&source); + computeDeltasCubeParallel(geo,currProjNumber_global,&xyzOrigin,&deltaX,&deltaY,&deltaZ,&source); + + offOrig.x=geo.offOrigX[currProjNumber_global]; + offOrig.y=geo.offOrigY[currProjNumber_global]; + + + projParamsArrayHostParallel[6*j]=deltaX; // 6*j because we have 6 Point3D values per projection + projParamsArrayHostParallel[6*j+1]=deltaY; + projParamsArrayHostParallel[6*j+2]=deltaZ; + projParamsArrayHostParallel[6*j+3]=xyzOrigin; + projParamsArrayHostParallel[6*j+4]=offOrig; + projParamsArrayHostParallel[6*j+5]=source; + } // END for (preparing params for kernel call) + + // Copy the prepared parameter arrays to constant memory to make it available for the kernel + + hipMemcpyToSymbolAsync(projSinCosArrayDevParallel, projSinCosArrayHostParallel, sizeof(float)*3*PROJ_PER_KERNEL,0,hipMemcpyHostToDevice,stream[0]); + hipMemcpyToSymbolAsync(projParamsArrayDevParallel, projParamsArrayHostParallel, sizeof(Point3D)*6*PROJ_PER_KERNEL,0,hipMemcpyHostToDevice,stream[0]); + hipStreamSynchronize(stream[0]); + + kernelPixelBackprojection_parallel<<>>(geo,dimage,i,proj_split_size[proj_block_split],texProj[(proj_block_split%2)]); + } // END for + + ////////////////////////////////////////////////////////////////////////////////////// + // END Main reconstruction loop: go through projections (rotation angles) and backproject + ////////////////////////////////////////////////////////////////////////////////////// + } + hipDeviceSynchronize(); + hipMemcpy(result, dimage, num_bytes, hipMemcpyDeviceToHost); + hipCheckErrors("hipMemcpy result fail"); + + free(partial_projection); + free(proj_split_size); + + bool two_buffers_used=((((nalpha+split_projections-1)/split_projections)+PROJ_PER_KERNEL-1)/PROJ_PER_KERNEL)>1; + for(unsigned int i=0; i<2;i++){ // 2 buffers (if needed, maybe only 1) + if (!two_buffers_used && i==1) + break; + hipDestroyTextureObject(texProj[i]); + hipFreeArray(d_cuArrTex[i]); + } + free(texProj); + + free(d_cuArrTex); + hipFreeHost(projSinCosArrayHostParallel); + hipFreeHost(projParamsArrayHostParallel); + + hipFree(dimage); + if (isHostRegisterSupported){ + hipHostUnregister(projections); + } + for (int i = 0; i < nStreams; ++i) + hipStreamDestroy(stream[i]); + +// hipDeviceReset(); + return 0; + +} // END voxel_backprojection + +void computeDeltasCubeParallel(Geometry geo, int i, Point3D* xyzorigin, Point3D* deltaX, Point3D* deltaY, Point3D* deltaZ,Point3D *S) +{ + + Point3Ddouble P, Px,Py,Pz; + // Get coords of Img(0,0,0) + P.x=-(geo.sVoxelX/2-geo.dVoxelX/2)+geo.offOrigX[i]; + P.y=-(geo.sVoxelY/2-geo.dVoxelY/2)+geo.offOrigY[i]; + P.z=-(geo.sVoxelZ/2-geo.dVoxelZ/2)+geo.offOrigZ[i]; + + // Get coors from next voxel in each direction + Px.x=P.x+geo.dVoxelX; Py.x=P.x; Pz.x=P.x; + Px.y=P.y; Py.y=P.y+geo.dVoxelY; Pz.y=P.y; + Px.z=P.z; Py.z=P.z; Pz.z=P.z+geo.dVoxelZ; + + + + // Rotate image around X axis (this is equivalent of rotating the source and detector) RZ RY RZ + eulerZYZT(geo,&P); + eulerZYZT(geo,&Px); + eulerZYZT(geo,&Py); + eulerZYZT(geo,&Pz); + + //detector offset + P.z =P.z-geo.offDetecV[i]; P.y =P.y-geo.offDetecU[i]; + Px.z =Px.z-geo.offDetecV[i]; Px.y =Px.y-geo.offDetecU[i]; + Py.z =Py.z-geo.offDetecV[i]; Py.y =Py.y-geo.offDetecU[i]; + Pz.z =Pz.z-geo.offDetecV[i]; Pz.y =Pz.y-geo.offDetecU[i]; + + //Detector Roll pitch Yaw + // + // + // first, we need to offset everything so (0,0,0) is the center of the detector + // Only X is required for that + P.x=P.x+(geo.DSD[i]-geo.DSO[i]); + Px.x=Px.x+(geo.DSD[i]-geo.DSO[i]); + Py.x=Py.x+(geo.DSD[i]-geo.DSO[i]); + Pz.x=Pz.x+(geo.DSD[i]-geo.DSO[i]); + + rollPitchYawT(geo,i,&P); + rollPitchYawT(geo,i,&Px); + rollPitchYawT(geo,i,&Py); + rollPitchYawT(geo,i,&Pz); + + P.x=P.x-(geo.DSD[i]-geo.DSO[i]); + Px.x=Px.x-(geo.DSD[i]-geo.DSO[i]); + Py.x=Py.x-(geo.DSD[i]-geo.DSO[i]); + Pz.x=Pz.x-(geo.DSD[i]-geo.DSO[i]); + + + Point3Ddouble source; + source.x=0; + source.y=-geo.offDetecU[i]; + source.z=-geo.offDetecV[i]; + + rollPitchYawT(geo,i,&source); + source.x=source.x-(geo.DSD[i]-geo.DSO[i]); + + P.z =P.z /geo.dDetecV; P.y =P.y/geo.dDetecU; + Px.z=Px.z/geo.dDetecV; Px.y=Px.y/geo.dDetecU; + Py.z=Py.z/geo.dDetecV; Py.y=Py.y/geo.dDetecU; + Pz.z=Pz.z/geo.dDetecV; Pz.y=Pz.y/geo.dDetecU; + + source.z=source.z/geo.dDetecV; source.y=source.y/geo.dDetecU; + + // get deltas of the changes in voxels + deltaX->x=Px.x-P.x; deltaX->y=Px.y-P.y; deltaX->z=Px.z-P.z; + deltaY->x=Py.x-P.x; deltaY->y=Py.y-P.y; deltaY->z=Py.z-P.z; + deltaZ->x=Pz.x-P.x; deltaZ->y=Pz.y-P.y; deltaZ->z=Pz.z-P.z; + + + // cast the results from the double precision calculations back to float + *xyzorigin=P.to_float(); + *S=source.to_float(); + + +} // END computeDeltasCube +void CreateTextureParallel(float* projectiondata,Geometry geo,hipArray** d_cuArrTex,unsigned int nangles, hipTextureObject_t *texImage,hipStream_t* stream, bool alloc) +{ + //hipArray Descriptor +#if IS_FOR_MATLAB_TIGRE + const hipExtent extent =make_cudaExtent(geo.nDetecV, geo.nDetecU, nangles); +#else + const hipExtent extent =make_cudaExtent(geo.nDetecU, geo.nDetecV, nangles); +#endif + hipChannelFormatDesc channelDesc = hipCreateChannelDesc(); + //cuda Array + if (alloc){ + hipMalloc3DArray(&d_cuArrTex[0], &channelDesc, extent); + hipCheckErrors("Texture memory allocation fail"); + } + hipMemcpy3DParms copyParams = {0}; + + + //Array creation + copyParams.srcPtr = make_cudaPitchedPtr((void *)projectiondata, extent.width*sizeof(float), extent.width, extent.height); + copyParams.dstArray = d_cuArrTex[0]; + copyParams.extent = extent; + copyParams.kind = hipMemcpyHostToDevice; + hipMemcpy3DAsync(©Params,stream[0+1]); + hipCheckErrors("Texture memory data copy fail"); + //Array creation End + + hipResourceDesc texRes; + memset(&texRes, 0, sizeof(hipResourceDesc)); + texRes.resType = hipResourceTypeArray; + texRes.res.array.array = d_cuArrTex[0]; + hipTextureDesc texDescr; + memset(&texDescr, 0, sizeof(hipTextureDesc)); + texDescr.normalizedCoords = false; + texDescr.filterMode = hipFilterModeLinear; + texDescr.addressMode[0] = hipAddressModeBorder; + texDescr.addressMode[1] = hipAddressModeBorder; + texDescr.addressMode[2] = hipAddressModeBorder; + texDescr.readMode = hipReadModeElementType; + hipCreateTextureObject(&texImage[0], &texRes, &texDescr, NULL); + hipCheckErrors("Texture object creation fail"); + } \ No newline at end of file diff --git a/Common/CUDA/voxel_backprojection_parallel.hpp b/Common/CUDA/voxel_backprojection_parallel.hpp index 92b72023..596e0c36 100644 --- a/Common/CUDA/voxel_backprojection_parallel.hpp +++ b/Common/CUDA/voxel_backprojection_parallel.hpp @@ -1,57 +1,57 @@ -/*------------------------------------------------------------------------- - * - * Header CUDA function for backrpojection for parallel beam - * - * - * CODE by Ander Biguri - * Optimized and modified by RB - * ---------------------------------------------------------------------------- ---------------------------------------------------------------------------- -Copyright (c) 2015, University of Bath and CERN- European Organization for -Nuclear Research -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, -this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors -may be used to endorse or promote products derived from this software without -specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------- - -Contact: tigre.toolbox@gmail.com -Codes : https://github.com/CERN/TIGRE ---------------------------------------------------------------------------- - */ -#include "types_TIGRE.hpp" -#include "GpuIds.hpp" - - -#ifndef BACKPROJECTION_PARALLEL_HPP -#define BACKPROJECTION_PARALLEL_HPP - -int voxel_backprojection_parallel(float * projections, Geometry geo, float* result,float const * const alphas,int nalpha, const GpuIds& gpuids); -void computeDeltasCubeParallel(Geometry geo, int i, Point3D* xyzorigin, Point3D* deltaX, Point3D* deltaY, Point3D* deltaZ,Point3D *S); -void createGeoArrayParallel(unsigned int image_splits, Geometry geo,Geometry* geoArray, unsigned int nangles); -// void computeDeltasCube(Geometry geo, float alpha,int i, Point3D* xyzorigin, Point3D* deltaX, Point3D* deltaY, Point3D* deltaZ); +/*------------------------------------------------------------------------- + * + * Header CUDA function for backrpojection for parallel beam + * + * + * CODE by Ander Biguri + * Optimized and modified by RB + * +--------------------------------------------------------------------------- +--------------------------------------------------------------------------- +Copyright (c) 2015, University of Bath and CERN- European Organization for +Nuclear Research +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + --------------------------------------------------------------------------- + +Contact: tigre.toolbox@gmail.com +Codes : https://github.com/CERN/TIGRE +--------------------------------------------------------------------------- + */ +#include "types_TIGRE.hpp" +#include "GpuIds.hpp" + + +#ifndef BACKPROJECTION_PARALLEL_HPP +#define BACKPROJECTION_PARALLEL_HPP + +int voxel_backprojection_parallel(float * projections, Geometry geo, float* result,float const * const alphas,int nalpha, const GpuIds& gpuids); +void computeDeltasCubeParallel(Geometry geo, int i, Point3D* xyzorigin, Point3D* deltaX, Point3D* deltaY, Point3D* deltaZ,Point3D *S); +void createGeoArrayParallel(unsigned int image_splits, Geometry geo,Geometry* geoArray, unsigned int nangles); +// void computeDeltasCube(Geometry geo, float alpha,int i, Point3D* xyzorigin, Point3D* deltaX, Point3D* deltaY, Point3D* deltaZ); #endif \ No newline at end of file diff --git a/setup.py b/setup.py index 7da395e5..c9b8fc05 100644 --- a/setup.py +++ b/setup.py @@ -1,525 +1,525 @@ -import copy -import glob -import os -from os.path import join as pjoin -import re -import subprocess -import sys -import shutil -import re - -from Cython.Distutils import build_ext -import numpy -from setuptools import setup, find_packages, Extension - - -# Macros for compilation -define_macros=[("IS_FOR_PYTIGRE", None)] -no_pinned=False -if len(sys.argv)>2: - if "--no_pinned_memory" in sys.argv[2:] : - no_pinned=True - sys.argv.pop(sys.argv.index("--no_pinned_memory")) - -if no_pinned: - define_macros.append(("NO_PINNED_MEMORY",None)) - -IS_WINDOWS = sys.platform == "win32" - - -# Code from https://github.com/pytorch/pytorch/blob/master/torch/utils/cpp_extension.py - -COMPUTE_CAPABILITY_ARGS = [ - "--ptxas-options=-v", - "-c", - "--default-stream=per-thread", -] - - -def get_cuda_version(cuda_home): - """Locate the CUDA version""" - version_file = os.path.join(cuda_home, "version.txt") - try: - if os.path.isfile(version_file): - with open(version_file) as f: - version_str = f.readline().replace("\n", "").replace("\r", "") - return version_str.split(" ")[2][:4] - else: - version_str = subprocess.check_output( - [os.path.join(cuda_home, "bin", "nvcc"), "--version"] - ) - version_str = str(version_str).replace("\n", "").replace("\r", "") - idx = version_str.find("release") - return version_str[idx + len("release ") : idx + len("release ") + 4] - except: - raise RuntimeError("Cannot read cuda version file") - - -def get_cuda_cc(cuda_home): - """List nvcc Compute Capabilities""" - cc = subprocess.check_output( - [os.path.join(cuda_home, "bin", "nvcc"), "--list-gpu-arch"] - ) - return re.findall(r"\d+", str(cc)) - - -def locate_cuda(): - """Locate the CUDA environment on the system - - Returns a dict with keys 'home', 'include' and 'lib64' - and values giving the absolute path to each directory. - - Starts by looking for the CUDA_HOME or CUDA_PATH env variable. If not found, everything - is based on finding 'nvcc' in the PATH. - """ - # Guess #1 - cuda_home = os.environ.get("CUDA_HOME") or os.environ.get("CUDA_PATH") - if cuda_home is None: - # Guess #2 - try: - which = "where" if IS_WINDOWS else "which" - nvcc = subprocess.check_output([which, "nvcc"]).decode().rstrip("\r\n") - cuda_home = os.path.dirname(os.path.dirname(nvcc)) - except subprocess.CalledProcessError: - # Guess #3 - if IS_WINDOWS: - cuda_homes = glob.glob("C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v*.*") - if len(cuda_homes) == 0: - cuda_home = "" - else: - cuda_home = cuda_homes[0] - else: - cuda_home = "/usr/local/cuda" - if not os.path.exists(cuda_home): - cuda_home = None - version = get_cuda_version(cuda_home) - cudaconfig = { - "home": cuda_home, - "include": pjoin(cuda_home, "include"), - } - if IS_WINDOWS: - cudaconfig["lib64"]= pjoin(cuda_home, pjoin("lib", "x64")) - else: - lib64_path = pjoin(cuda_home, "lib64") - cudaconfig["lib64"] = lib64_path if os.path.exists(lib64_path) else pjoin(cuda_home, "lib") - - if not all([os.path.exists(v) for v in cudaconfig.values()]): - raise EnvironmentError( - "The CUDA path could not be located in $PATH, $CUDA_HOME or $CUDA_PATH. " - "Either add it to your path, or set $CUDA_HOME or $CUDA_PATH." - ) - - return cudaconfig, version - - -def _is_cuda_file(path): - return os.path.splitext(path)[1] in [".cu", ".cuh"] - - -CUDA, CUDA_VERSION = locate_cuda() - -cuda_version = 11.0 -try: - cuda_version = float(CUDA_VERSION) -except ValueError: - cuda_list = re.findall(r'\d+', CUDA_VERSION) - cuda_version = float( str(cuda_list[0] + '.' + cuda_list[1])) - -# Insert CUDA CC arguments -for item in get_cuda_cc(CUDA["home"]): - str_arg = f"-gencode=arch=compute_{item},code=sm_{item}" - COMPUTE_CAPABILITY_ARGS.insert(0, str_arg) - - -# Obtain the numpy include directory. This logic works across numpy versions. -try: - NUMPY_INCLUDE = numpy.get_include() -except AttributeError: - NUMPY_INCLUDE = numpy.get_numpy_include() - - -COMMON_MSVC_FLAGS = ["/MD", "/wd4819", "/EHsc"] - - -COMMON_NVCC_FLAGS = [ - "-D__CUDA_NO_HALF_OPERATORS__", - "-D__CUDA_NO_HALF_CONVERSIONS__", - "-D__CUDA_NO_HALF2_OPERATORS__", - "--expt-relaxed-constexpr", -] - - -def _join_cuda_home(*paths): - return os.path.join(CUDA["home"], *paths) - - -class BuildExtension(build_ext): - """ - A custom :mod:`Cython.Distutils` build extension . - - This :class:`Cython.Distutils.build_ext` subclass takes care of passing the - minimum required compiler flags (e.g. ``-std=c++11``) as well as mixed - C++/CUDA compilation (and support for CUDA files in general). - - When using :class:`BuildExtension`, it is allowed to supply a dictionary - for ``extra_compile_args`` (rather than the usual list) that maps from - languages (``cxx`` or ``nvcc``) to a list of additional compiler flags to - supply to the compiler. This makes it possible to supply different flags to - the C++ and CUDA compiler during mixed compilation. - """ - - @classmethod - def with_options(cls, **options): - """ - Returns an alternative constructor that extends any original keyword - arguments to the original constructor with the given options. - """ - - def init_with_options(*args, **kwargs): - kwargs = kwargs.copy() - kwargs.update(options) - return cls(*args, **kwargs) - - return init_with_options - - def __init__(self, *args, **kwargs): - build_ext.__init__(self, *args, **kwargs) - self.no_python_abi_suffix = kwargs.get("no_python_abi_suffix", False) - - def build_extensions(self): - # Register .cu and .cuh as valid source extensions. - self.compiler.src_extensions += [".cu", ".cuh"] - # Save the original _compile method for later. - if self.compiler.compiler_type == "msvc": - self.compiler._cpp_extensions += [".cu", ".cuh"] - original_compile = self.compiler.compile - original_spawn = self.compiler.spawn - else: - original_compile = self.compiler._compile - - def unix_wrap_compile(obj, src, ext, cc_args, extra_postargs, pp_opts): - # Copy before we make any modifications. - cflags = copy.deepcopy(extra_postargs) - try: - original_compiler = self.compiler.compiler_so - if _is_cuda_file(src): - nvcc = _join_cuda_home("bin", "nvcc") - if not isinstance(nvcc, list): - nvcc = [nvcc] - self.compiler.set_executable("compiler_so", nvcc) - if isinstance(cflags, dict): - cflags = cflags["nvcc"] - cflags = ( - COMMON_NVCC_FLAGS - + ["--compiler-options", "'-fPIC'"] - + cflags - + COMPUTE_CAPABILITY_ARGS - ) - elif isinstance(cflags, dict): - cflags = cflags["cxx"] - # NVCC does not allow multiple -std to be passed, so we avoid - # overriding the option if the user explicitly passed it. - if not any(flag.startswith("-std=") for flag in cflags): - cflags.append("-std=c++11") - - original_compile(obj, src, ext, cc_args, cflags, pp_opts) - finally: - # Put the original compiler back in place. - self.compiler.set_executable("compiler_so", original_compiler) - - def win_wrap_compile( - sources, - output_dir=None, - macros=None, - include_dirs=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - depends=None, - ): - - cflags = copy.deepcopy(extra_postargs) - extra_postargs = None - - def spawn(cmd, cflags): - # Using regex to match src, obj and include files - src_regex = re.compile("/T(p|c)(.*)") - src_list = [m.group(2) for m in (src_regex.match(elem) for elem in cmd) if m] - - obj_regex = re.compile("/Fo(.*)") - obj_list = [m.group(1) for m in (obj_regex.match(elem) for elem in cmd) if m] - - include_regex = re.compile(r"((\-|\/)I.*)") - include_list = [ - m.group(1) for m in (include_regex.match(elem) for elem in cmd) if m - ] - - if len(src_list) >= 1 and len(obj_list) >= 1: - src = src_list[0] - obj = obj_list[0] - if _is_cuda_file(src): - nvcc = _join_cuda_home("bin", "nvcc") - if isinstance(cflags, dict): - cflags = cflags["nvcc"] - elif not isinstance(cflags, list): - cflags = [] - - cflags = COMMON_NVCC_FLAGS + cflags + COMPUTE_CAPABILITY_ARGS - for flag in COMMON_MSVC_FLAGS: - cflags = ["-Xcompiler", flag] + cflags - for macro in macros: - if len(macro) == 2: - if macro[1] == None: - cflags += ["--define-macro", macro[0]] - else: - cflags += ["--define-macro", "{}={}".format(macro[0], macro[1])] - elif len(macro) == 1: - cflags += ["--undefine-macro", macro[0]] - - cmd = [nvcc, "-c", src, "-o", obj] + include_list + cflags - elif isinstance(cflags, dict): - cflags = COMMON_MSVC_FLAGS # + self.cflags['cxx'] - cmd += cflags - elif isinstance(cflags, list): - cflags = COMMON_MSVC_FLAGS + cflags - cmd += cflags - - return original_spawn(cmd) - - try: - self.compiler.spawn = lambda cmd: spawn(cmd, cflags) - return original_compile( - sources, - output_dir, - macros, - include_dirs, - debug, - extra_preargs, - extra_postargs, - depends, - ) - finally: - self.compiler.spawn = original_spawn - - # Monkey-patch the _compile method. - if self.compiler.compiler_type == "msvc": - self.compiler.compile = win_wrap_compile - else: - self.compiler._compile = unix_wrap_compile - - build_ext.build_extensions(self) - - def get_ext_filename(self, ext_name): - # Get the original shared library name. For Python 3, this name will be - # suffixed with ".so", where will be something like - # cpython-37m-x86_64-linux-gnu. On Python 2, there is no such ABI name. - # The final extension, .so, would be .lib/.dll on Windows of course. - ext_filename = build_ext.get_ext_filename(self, ext_name) - # If `no_python_abi_suffix` is `True`, we omit the Python 3 ABI - # component. This makes building shared libraries with setuptools that - # aren't Python modules nicer. - if self.no_python_abi_suffix and sys.version_info >= (3, 0): - # The parts will be e.g. ["my_extension", "cpython-37m-x86_64-linux-gnu", "so"]. - ext_filename_parts = ext_filename.split(".") - # Omit the second to last element. - without_abi = ext_filename_parts[:-2] + ext_filename_parts[-1:] - ext_filename = ".".join(without_abi) - return ext_filename - - -def include_headers(filename_list, sdist=False): - """add hpp and h files to list if sdist is called""" - if not sdist: - return filename_list - - c_extensions = [".cu", ".c", ".C", ".cc", ".cpp", ".cxx", ".c++"] - header_list = [] - for filename in filename_list: - header = list(os.path.splitext(filename)) - if header[1] in c_extensions: - header[1] = ".hpp" - header_list.append("".join(header)) - - filename_list += ["Common/CUDA/types_TIGRE.hpp", "Common/CUDA/errors.hpp"] - return filename_list + header_list - - -Ax_ext = Extension( - "_Ax", - sources=include_headers( - [ - "Common/CUDA/projection.cpp", - "Common/CUDA/TIGRE_common.cpp", - "Common/CUDA/Siddon_projection.cu", - "Common/CUDA/Siddon_projection_parallel.cu", - "Common/CUDA/ray_interpolated_projection.cu", - "Common/CUDA/ray_interpolated_projection_parallel.cu", - "Common/CUDA/GpuIds.cpp", - "Python/tigre/utilities/cuda_interface/_Ax.pyx", - ], - sdist=sys.argv[1] == "sdist", - ), - define_macros=define_macros, - library_dirs=[CUDA["lib64"]], - libraries=["cudart"], - language="c++", - runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, - include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], -) - - -Atb_ext = Extension( - "_Atb", - sources=include_headers( - [ - "Common/CUDA/TIGRE_common.cpp", - "Common/CUDA/voxel_backprojection.cu", - "Common/CUDA/voxel_backprojection2.cu", - "Common/CUDA/voxel_backprojection_parallel.cu", - "Common/CUDA/GpuIds.cpp", - "Common/CUDA/gpuUtils.cu", - "Python/tigre/utilities/cuda_interface/_Atb.pyx", - ], - sdist=sys.argv[1] == "sdist", - ), - define_macros=define_macros, - library_dirs=[CUDA["lib64"]], - libraries=["cudart"], - language="c++", - runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, - include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], -) - - -tv_proximal_ext = Extension( - "_tv_proximal", - sources=include_headers( - [ - "Common/CUDA/TIGRE_common.cpp", - "Common/CUDA/tv_proximal.cu", - "Common/CUDA/GpuIds.cpp", - "Common/CUDA/gpuUtils.cu", - "Python/tigre/utilities/cuda_interface/_tv_proximal.pyx", - ], - sdist=sys.argv[1] == "sdist", - ), - define_macros=define_macros, - library_dirs=[CUDA["lib64"]], - libraries=["cudart"], - language="c++", - runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, - include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], -) - - -minTV_ext = Extension( - "_minTV", - sources=include_headers( - [ - "Common/CUDA/TIGRE_common.cpp", - "Common/CUDA/GD_TV.cu", - "Common/CUDA/GpuIds.cpp", - "Common/CUDA/gpuUtils.cu", - "Python/tigre/utilities/cuda_interface/_minTV.pyx", - ], - sdist=sys.argv[1] == "sdist", - ), - define_macros=define_macros, - library_dirs=[CUDA["lib64"]], - libraries=["cudart"], - language="c++", - runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, - include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], -) - -minPICCS_ext = Extension( - "_minPICCS", - sources=include_headers( - [ - "Common/CUDA/TIGRE_common.cpp", - "Common/CUDA/PICCS.cu", - "Common/CUDA/GpuIds.cpp", - "Common/CUDA/gpuUtils.cu", - "Python/tigre/utilities/cuda_interface/_minPICCS.pyx", - ], - sdist=sys.argv[1] == "sdist", - ), - define_macros=define_macros, - library_dirs=[CUDA["lib64"]], - libraries=["cudart"], - language="c++", - runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, - include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], -) - -AwminTV_ext = Extension( - "_AwminTV", - sources=include_headers( - [ - "Common/CUDA/TIGRE_common.cpp", - "Common/CUDA/GD_AwTV.cu", - "Common/CUDA/GpuIds.cpp", - "Common/CUDA/gpuUtils.cu", - "Python/tigre/utilities/cuda_interface/_AwminTV.pyx", - ], - sdist=sys.argv[1] == "sdist", - ), - define_macros=define_macros, - library_dirs=[CUDA["lib64"]], - libraries=["cudart"], - language="c++", - runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, - include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], -) - - -gpuUtils_ext = Extension( - "_gpuUtils", - sources=include_headers( - [ - "Common/CUDA/gpuUtils.cu", - "Python/tigre/utilities/cuda_interface/_gpuUtils.pyx", - ], - sdist=sys.argv[1] == "sdist", - ), - library_dirs=[CUDA["lib64"]], - libraries=["cudart"], - language="c++", - runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, - include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], -) - - -RandomNumberGenerator_ext = Extension( - "_RandomNumberGenerator", - sources=include_headers( - [ - "Common/CUDA/TIGRE_common.cpp", - "Common/CUDA/RandomNumberGenerator.cu", - "Common/CUDA/GpuIds.cpp", - "Common/CUDA/gpuUtils.cu", - "Python/tigre/utilities/cuda_interface/_randomNumberGenerator.pyx", - ], - sdist=sys.argv[1] == "sdist", - ), - define_macros=define_macros, - library_dirs=[CUDA["lib64"]], - libraries=["cudart"], - language="c++", - runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, - include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], -) - -print(find_packages()) - -setup( - packages=find_packages(), - include_package_data=True, - data_files=[("data", ["Common/data/head.mat"])], - ext_modules=[minPICCS_ext, Ax_ext, Atb_ext, tv_proximal_ext, minTV_ext, AwminTV_ext, gpuUtils_ext, RandomNumberGenerator_ext], - py_modules=["tigre.py"], - cmdclass={"build_ext": BuildExtension}, - # since the package has c code, the egg cannot be zipped - zip_safe=False, -) - +import copy +import glob +import os +from os.path import join as pjoin +import re +import subprocess +import sys +import shutil +import re + +from Cython.Distutils import build_ext +import numpy +from setuptools import setup, find_packages, Extension + + +# Macros for compilation +define_macros=[("IS_FOR_PYTIGRE", None)] +no_pinned=False +if len(sys.argv)>2: + if "--no_pinned_memory" in sys.argv[2:] : + no_pinned=True + sys.argv.pop(sys.argv.index("--no_pinned_memory")) + +if no_pinned: + define_macros.append(("NO_PINNED_MEMORY",None)) + +IS_WINDOWS = sys.platform == "win32" + + +# Code from https://github.com/pytorch/pytorch/blob/master/torch/utils/cpp_extension.py + +COMPUTE_CAPABILITY_ARGS = [ + "--ptxas-options=-v", + "-c", + "--default-stream=per-thread", +] + + +def get_cuda_version(cuda_home): + """Locate the CUDA version""" + version_file = os.path.join(cuda_home, "version.txt") + try: + if os.path.isfile(version_file): + with open(version_file) as f: + version_str = f.readline().replace("\n", "").replace("\r", "") + return version_str.split(" ")[2][:4] + else: + version_str = subprocess.check_output( + [os.path.join(cuda_home, "bin", "nvcc"), "--version"] + ) + version_str = str(version_str).replace("\n", "").replace("\r", "") + idx = version_str.find("release") + return version_str[idx + len("release ") : idx + len("release ") + 4] + except: + raise RuntimeError("Cannot read cuda version file") + + +def get_cuda_cc(cuda_home): + """List nvcc Compute Capabilities""" + cc = subprocess.check_output( + [os.path.join(cuda_home, "bin", "nvcc"), "--list-gpu-arch"] + ) + return re.findall(r"\d+", str(cc)) + + +def locate_cuda(): + """Locate the CUDA environment on the system + + Returns a dict with keys 'home', 'include' and 'lib64' + and values giving the absolute path to each directory. + + Starts by looking for the CUDA_HOME or CUDA_PATH env variable. If not found, everything + is based on finding 'nvcc' in the PATH. + """ + # Guess #1 + cuda_home = os.environ.get("CUDA_HOME") or os.environ.get("CUDA_PATH") + if cuda_home is None: + # Guess #2 + try: + which = "where" if IS_WINDOWS else "which" + nvcc = subprocess.check_output([which, "nvcc"]).decode().rstrip("\r\n") + cuda_home = os.path.dirname(os.path.dirname(nvcc)) + except subprocess.CalledProcessError: + # Guess #3 + if IS_WINDOWS: + cuda_homes = glob.glob("C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v*.*") + if len(cuda_homes) == 0: + cuda_home = "" + else: + cuda_home = cuda_homes[0] + else: + cuda_home = "/usr/local/cuda" + if not os.path.exists(cuda_home): + cuda_home = None + version = get_cuda_version(cuda_home) + cudaconfig = { + "home": cuda_home, + "include": pjoin(cuda_home, "include"), + } + if IS_WINDOWS: + cudaconfig["lib64"]= pjoin(cuda_home, pjoin("lib", "x64")) + else: + lib64_path = pjoin(cuda_home, "lib64") + cudaconfig["lib64"] = lib64_path if os.path.exists(lib64_path) else pjoin(cuda_home, "lib") + + if not all([os.path.exists(v) for v in cudaconfig.values()]): + raise EnvironmentError( + "The CUDA path could not be located in $PATH, $CUDA_HOME or $CUDA_PATH. " + "Either add it to your path, or set $CUDA_HOME or $CUDA_PATH." + ) + + return cudaconfig, version + + +def _is_cuda_file(path): + return os.path.splitext(path)[1] in [".cu", ".cuh"] + + +CUDA, CUDA_VERSION = locate_cuda() + +cuda_version = 11.0 +try: + cuda_version = float(CUDA_VERSION) +except ValueError: + cuda_list = re.findall(r'\d+', CUDA_VERSION) + cuda_version = float( str(cuda_list[0] + '.' + cuda_list[1])) + +# Insert CUDA CC arguments +for item in get_cuda_cc(CUDA["home"]): + str_arg = f"-gencode=arch=compute_{item},code=sm_{item}" + COMPUTE_CAPABILITY_ARGS.insert(0, str_arg) + + +# Obtain the numpy include directory. This logic works across numpy versions. +try: + NUMPY_INCLUDE = numpy.get_include() +except AttributeError: + NUMPY_INCLUDE = numpy.get_numpy_include() + + +COMMON_MSVC_FLAGS = ["/MD", "/wd4819", "/EHsc"] + + +COMMON_NVCC_FLAGS = [ + "-D__CUDA_NO_HALF_OPERATORS__", + "-D__CUDA_NO_HALF_CONVERSIONS__", + "-D__CUDA_NO_HALF2_OPERATORS__", + "--expt-relaxed-constexpr", +] + + +def _join_cuda_home(*paths): + return os.path.join(CUDA["home"], *paths) + + +class BuildExtension(build_ext): + """ + A custom :mod:`Cython.Distutils` build extension . + + This :class:`Cython.Distutils.build_ext` subclass takes care of passing the + minimum required compiler flags (e.g. ``-std=c++11``) as well as mixed + C++/CUDA compilation (and support for CUDA files in general). + + When using :class:`BuildExtension`, it is allowed to supply a dictionary + for ``extra_compile_args`` (rather than the usual list) that maps from + languages (``cxx`` or ``nvcc``) to a list of additional compiler flags to + supply to the compiler. This makes it possible to supply different flags to + the C++ and CUDA compiler during mixed compilation. + """ + + @classmethod + def with_options(cls, **options): + """ + Returns an alternative constructor that extends any original keyword + arguments to the original constructor with the given options. + """ + + def init_with_options(*args, **kwargs): + kwargs = kwargs.copy() + kwargs.update(options) + return cls(*args, **kwargs) + + return init_with_options + + def __init__(self, *args, **kwargs): + build_ext.__init__(self, *args, **kwargs) + self.no_python_abi_suffix = kwargs.get("no_python_abi_suffix", False) + + def build_extensions(self): + # Register .cu and .cuh as valid source extensions. + self.compiler.src_extensions += [".cu", ".cuh"] + # Save the original _compile method for later. + if self.compiler.compiler_type == "msvc": + self.compiler._cpp_extensions += [".cu", ".cuh"] + original_compile = self.compiler.compile + original_spawn = self.compiler.spawn + else: + original_compile = self.compiler._compile + + def unix_wrap_compile(obj, src, ext, cc_args, extra_postargs, pp_opts): + # Copy before we make any modifications. + cflags = copy.deepcopy(extra_postargs) + try: + original_compiler = self.compiler.compiler_so + if _is_cuda_file(src): + nvcc = os.environ.get("TIGRE_HIPCC", _join_cuda_home("bin", "nvcc")) + if not isinstance(nvcc, list): + nvcc = [nvcc] + self.compiler.set_executable("compiler_so", nvcc) + if isinstance(cflags, dict): + cflags = cflags["nvcc"] + cflags = ( + COMMON_NVCC_FLAGS + + ["--compiler-options", "'-fPIC'"] + + cflags + + COMPUTE_CAPABILITY_ARGS + ) + elif isinstance(cflags, dict): + cflags = cflags["cxx"] + # NVCC does not allow multiple -std to be passed, so we avoid + # overriding the option if the user explicitly passed it. + if not any(flag.startswith("-std=") for flag in cflags): + cflags.append("-std=c++11") + + original_compile(obj, src, ext, cc_args, cflags, pp_opts) + finally: + # Put the original compiler back in place. + self.compiler.set_executable("compiler_so", original_compiler) + + def win_wrap_compile( + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None, + ): + + cflags = copy.deepcopy(extra_postargs) + extra_postargs = None + + def spawn(cmd, cflags): + # Using regex to match src, obj and include files + src_regex = re.compile("/T(p|c)(.*)") + src_list = [m.group(2) for m in (src_regex.match(elem) for elem in cmd) if m] + + obj_regex = re.compile("/Fo(.*)") + obj_list = [m.group(1) for m in (obj_regex.match(elem) for elem in cmd) if m] + + include_regex = re.compile(r"((\-|\/)I.*)") + include_list = [ + m.group(1) for m in (include_regex.match(elem) for elem in cmd) if m + ] + + if len(src_list) >= 1 and len(obj_list) >= 1: + src = src_list[0] + obj = obj_list[0] + if _is_cuda_file(src): + nvcc = os.environ.get("TIGRE_HIPCC", _join_cuda_home("bin", "nvcc")) + if isinstance(cflags, dict): + cflags = cflags["nvcc"] + elif not isinstance(cflags, list): + cflags = [] + + cflags = COMMON_NVCC_FLAGS + cflags + COMPUTE_CAPABILITY_ARGS + for flag in COMMON_MSVC_FLAGS: + cflags = ["-Xcompiler", flag] + cflags + for macro in macros: + if len(macro) == 2: + if macro[1] == None: + cflags += ["--define-macro", macro[0]] + else: + cflags += ["--define-macro", "{}={}".format(macro[0], macro[1])] + elif len(macro) == 1: + cflags += ["--undefine-macro", macro[0]] + + cmd = [nvcc, "-c", src, "-o", obj] + include_list + cflags + elif isinstance(cflags, dict): + cflags = COMMON_MSVC_FLAGS # + self.cflags['cxx'] + cmd += cflags + elif isinstance(cflags, list): + cflags = COMMON_MSVC_FLAGS + cflags + cmd += cflags + + return original_spawn(cmd) + + try: + self.compiler.spawn = lambda cmd: spawn(cmd, cflags) + return original_compile( + sources, + output_dir, + macros, + include_dirs, + debug, + extra_preargs, + extra_postargs, + depends, + ) + finally: + self.compiler.spawn = original_spawn + + # Monkey-patch the _compile method. + if self.compiler.compiler_type == "msvc": + self.compiler.compile = win_wrap_compile + else: + self.compiler._compile = unix_wrap_compile + + build_ext.build_extensions(self) + + def get_ext_filename(self, ext_name): + # Get the original shared library name. For Python 3, this name will be + # suffixed with ".so", where will be something like + # cpython-37m-x86_64-linux-gnu. On Python 2, there is no such ABI name. + # The final extension, .so, would be .lib/.dll on Windows of course. + ext_filename = build_ext.get_ext_filename(self, ext_name) + # If `no_python_abi_suffix` is `True`, we omit the Python 3 ABI + # component. This makes building shared libraries with setuptools that + # aren't Python modules nicer. + if self.no_python_abi_suffix and sys.version_info >= (3, 0): + # The parts will be e.g. ["my_extension", "cpython-37m-x86_64-linux-gnu", "so"]. + ext_filename_parts = ext_filename.split(".") + # Omit the second to last element. + without_abi = ext_filename_parts[:-2] + ext_filename_parts[-1:] + ext_filename = ".".join(without_abi) + return ext_filename + + +def include_headers(filename_list, sdist=False): + """add hpp and h files to list if sdist is called""" + if not sdist: + return filename_list + + c_extensions = [".cu", ".c", ".C", ".cc", ".cpp", ".cxx", ".c++"] + header_list = [] + for filename in filename_list: + header = list(os.path.splitext(filename)) + if header[1] in c_extensions: + header[1] = ".hpp" + header_list.append("".join(header)) + + filename_list += ["Common/CUDA/types_TIGRE.hpp", "Common/CUDA/errors.hpp"] + return filename_list + header_list + + +Ax_ext = Extension( + "_Ax", + sources=include_headers( + [ + "Common/CUDA/projection.cpp", + "Common/CUDA/TIGRE_common.cpp", + "Common/CUDA/Siddon_projection.cu", + "Common/CUDA/Siddon_projection_parallel.cu", + "Common/CUDA/ray_interpolated_projection.cu", + "Common/CUDA/ray_interpolated_projection_parallel.cu", + "Common/CUDA/GpuIds.cpp", + "Python/tigre/utilities/cuda_interface/_Ax.pyx", + ], + sdist=sys.argv[1] == "sdist", + ), + define_macros=define_macros, + library_dirs=[CUDA["lib64"]], + libraries=["amdhip64"] if os.environ.get("TIGRE_HIPCC") else ["cudart"], + language="c++", + runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, + include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], +) + + +Atb_ext = Extension( + "_Atb", + sources=include_headers( + [ + "Common/CUDA/TIGRE_common.cpp", + "Common/CUDA/voxel_backprojection.cu", + "Common/CUDA/voxel_backprojection2.cu", + "Common/CUDA/voxel_backprojection_parallel.cu", + "Common/CUDA/GpuIds.cpp", + "Common/CUDA/gpuUtils.cu", + "Python/tigre/utilities/cuda_interface/_Atb.pyx", + ], + sdist=sys.argv[1] == "sdist", + ), + define_macros=define_macros, + library_dirs=[CUDA["lib64"]], + libraries=["amdhip64"] if os.environ.get("TIGRE_HIPCC") else ["cudart"], + language="c++", + runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, + include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], +) + + +tv_proximal_ext = Extension( + "_tv_proximal", + sources=include_headers( + [ + "Common/CUDA/TIGRE_common.cpp", + "Common/CUDA/tv_proximal.cu", + "Common/CUDA/GpuIds.cpp", + "Common/CUDA/gpuUtils.cu", + "Python/tigre/utilities/cuda_interface/_tv_proximal.pyx", + ], + sdist=sys.argv[1] == "sdist", + ), + define_macros=define_macros, + library_dirs=[CUDA["lib64"]], + libraries=["amdhip64"] if os.environ.get("TIGRE_HIPCC") else ["cudart"], + language="c++", + runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, + include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], +) + + +minTV_ext = Extension( + "_minTV", + sources=include_headers( + [ + "Common/CUDA/TIGRE_common.cpp", + "Common/CUDA/GD_TV.cu", + "Common/CUDA/GpuIds.cpp", + "Common/CUDA/gpuUtils.cu", + "Python/tigre/utilities/cuda_interface/_minTV.pyx", + ], + sdist=sys.argv[1] == "sdist", + ), + define_macros=define_macros, + library_dirs=[CUDA["lib64"]], + libraries=["amdhip64"] if os.environ.get("TIGRE_HIPCC") else ["cudart"], + language="c++", + runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, + include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], +) + +minPICCS_ext = Extension( + "_minPICCS", + sources=include_headers( + [ + "Common/CUDA/TIGRE_common.cpp", + "Common/CUDA/PICCS.cu", + "Common/CUDA/GpuIds.cpp", + "Common/CUDA/gpuUtils.cu", + "Python/tigre/utilities/cuda_interface/_minPICCS.pyx", + ], + sdist=sys.argv[1] == "sdist", + ), + define_macros=define_macros, + library_dirs=[CUDA["lib64"]], + libraries=["amdhip64"] if os.environ.get("TIGRE_HIPCC") else ["cudart"], + language="c++", + runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, + include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], +) + +AwminTV_ext = Extension( + "_AwminTV", + sources=include_headers( + [ + "Common/CUDA/TIGRE_common.cpp", + "Common/CUDA/GD_AwTV.cu", + "Common/CUDA/GpuIds.cpp", + "Common/CUDA/gpuUtils.cu", + "Python/tigre/utilities/cuda_interface/_AwminTV.pyx", + ], + sdist=sys.argv[1] == "sdist", + ), + define_macros=define_macros, + library_dirs=[CUDA["lib64"]], + libraries=["amdhip64"] if os.environ.get("TIGRE_HIPCC") else ["cudart"], + language="c++", + runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, + include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], +) + + +gpuUtils_ext = Extension( + "_gpuUtils", + sources=include_headers( + [ + "Common/CUDA/gpuUtils.cu", + "Python/tigre/utilities/cuda_interface/_gpuUtils.pyx", + ], + sdist=sys.argv[1] == "sdist", + ), + library_dirs=[CUDA["lib64"]], + libraries=["amdhip64"] if os.environ.get("TIGRE_HIPCC") else ["cudart"], + language="c++", + runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, + include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], +) + + +RandomNumberGenerator_ext = Extension( + "_RandomNumberGenerator", + sources=include_headers( + [ + "Common/CUDA/TIGRE_common.cpp", + "Common/CUDA/RandomNumberGenerator.cu", + "Common/CUDA/GpuIds.cpp", + "Common/CUDA/gpuUtils.cu", + "Python/tigre/utilities/cuda_interface/_randomNumberGenerator.pyx", + ], + sdist=sys.argv[1] == "sdist", + ), + define_macros=define_macros, + library_dirs=[CUDA["lib64"]], + libraries=["amdhip64"] if os.environ.get("TIGRE_HIPCC") else ["cudart"], + language="c++", + runtime_library_dirs=[CUDA["lib64"]] if not IS_WINDOWS else None, + include_dirs=[NUMPY_INCLUDE, CUDA["include"], "Common/CUDA/"], +) + +print(find_packages()) + +setup( + packages=find_packages(), + include_package_data=True, + data_files=[("data", ["Common/data/head.mat"])], + ext_modules=[minPICCS_ext, Ax_ext, Atb_ext, tv_proximal_ext, minTV_ext, AwminTV_ext, gpuUtils_ext, RandomNumberGenerator_ext], + py_modules=["tigre.py"], + cmdclass={"build_ext": BuildExtension}, + # since the package has c code, the egg cannot be zipped + zip_safe=False, +) +