From a63ee36f4cfa144842910068bd5418b3ec00a485 Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Fri, 4 Aug 2017 14:45:26 +0800 Subject: [PATCH 01/14] Add files via upload Try to add some switch options to selected running CPU or GPU (using OpenCL) --- src/libopentld/tld/switch.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/libopentld/tld/switch.h diff --git a/src/libopentld/tld/switch.h b/src/libopentld/tld/switch.h new file mode 100644 index 00000000..ae7f5446 --- /dev/null +++ b/src/libopentld/tld/switch.h @@ -0,0 +1,14 @@ +#define opencl 1 + +#define clDetect 1 +#define clIntegral 1 +#define clIntegral_m 1 +#define clIntegral_m_1 0 +#define cloverlap 0 + +#define PrintTime_overlap 0 +#define PrintTime_detect 0 +#define PrintTime_learn 0 +#define PrintTime_integral 0 +#define PrintTime_fuseHypotheses 0 +#define SELECTED_DEVICE_ID 0 \ No newline at end of file From 2fb591bd642da83823908cb6a37f64df7890b57c Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Fri, 4 Aug 2017 14:59:57 +0800 Subject: [PATCH 02/14] Update README.md Begin to learn github by Completing OpenTLD using OpenCL. --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a46ea226..9ff0ce46 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ This is a C++ implementation of OpenTLD that was originally published in MATLAB by Zdenek Kalal. OpenTLD is used for tracking objects in video streams. What makes this algorithm outstanding is that it does not make use of any training data. This implementation is based solely on open source libraries, meaning that you do not need any commercial products to compile or run it. -The easiest way to get started is to download the precompiled [binaries](https://github.com/gnebehay/OpenTLD/releases) that are available for Windows and Ubuntu 12.04. +GPU is a good accelerated compute resource for speed up Tracking speed, so I want to complete one OpenCL version for OpenTLD, I will begin today. + +The easiest way to get started is to download the precompiled [binaries](https://github.com/gnebehay/OpenTLD/releases) that are available for Windows and Ubuntu 12.04.   There is also a [PPA](https://launchpad.net/~opentld/+archive/ppa) available for Ubuntu 12.04. You just have to execute these commands: ``` sudo add-apt-repository ppa:opentld/ppa From 52f9ed2d2d196dbcebb70b53d4bee7ca379c79a5 Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Mon, 11 Sep 2017 10:16:29 +0800 Subject: [PATCH 03/14] changes method to get Video fream imAcq->method = IMACQ_CAM; --- src/libopentld/imacq/ImAcq.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libopentld/imacq/ImAcq.cpp b/src/libopentld/imacq/ImAcq.cpp index 24d44410..9cd33467 100644 --- a/src/libopentld/imacq/ImAcq.cpp +++ b/src/libopentld/imacq/ImAcq.cpp @@ -42,7 +42,8 @@ static void msleep(int milliseconds) ImAcq *imAcqAlloc() { ImAcq *imAcq = (ImAcq *)malloc(sizeof(ImAcq)); - imAcq->method = IMACQ_CAM; + // imAcq->method = IMACQ_CAM; + imAcq->method = 2; imAcq->currentFrame = 1; imAcq->lastFrame = 0; imAcq->camNo = 0; @@ -52,6 +53,7 @@ ImAcq *imAcqAlloc() void imAcqInit(ImAcq *imAcq) { + // imAcq->method = 2; if(imAcq->method == IMACQ_CAM) { imAcq->capture = cvCaptureFromCAM(imAcq->camNo); @@ -64,7 +66,8 @@ void imAcqInit(ImAcq *imAcq) } else if(imAcq->method == IMACQ_VID) { - imAcq->capture = cvCaptureFromAVI(imAcq->imgPath); + imAcq->capture = cvCaptureFromAVI(imAcq->imgPath); + //imAcq->capture = cvCaptureFromAVI("D:\\OpenTLD-master\\build\\bin\\Release\\IMG_5903.MOV"); if(imAcq->capture == NULL) { @@ -283,3 +286,4 @@ int imAcqVidGetNumberOfFrames(ImAcq *imAcq) { return ((int) cvGetCaptureProperty(imAcq->capture , CV_CAP_PROP_FRAME_COUNT)); } + From 2777f83559c19f3178a9c5d8f6947b1ec7fedcf1 Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Mon, 11 Sep 2017 13:29:43 +0800 Subject: [PATCH 04/14] Add 3 OpenCL kernels to repository Add 3 OpenCL kernels to repository 1. cltldOverlapRect_kernel.cpp 2. EnsembleClassifierFilter_kernel.cl.cpp 3. varianceFilter_kernel.cpp --- .../EnsembleClassifierFilter_kernel.cl.cpp | 368 ++++++++++++++++++ .../kernel/cltldOverlapRect_kernel.cpp | 76 ++++ .../kernel/varianceFilter_kernel.cpp | 46 +++ 3 files changed, 490 insertions(+) create mode 100644 src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp create mode 100644 src/libopentld/kernel/cltldOverlapRect_kernel.cpp create mode 100644 src/libopentld/kernel/varianceFilter_kernel.cpp diff --git a/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp new file mode 100644 index 00000000..10a45bd7 --- /dev/null +++ b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp @@ -0,0 +1,368 @@ + + +#pragma OPENCL EXTENSION cl_amd_printf : enable + + + +__kernel void EnsembleClassifierFilter( + const int numIndices, + const unsigned int numTrees, + const unsigned int numFeatures, + const unsigned int TLD_WINDOW_OFFSET_SIZE, + __global int* oclbuffWindowsOffset, + __global uchar * img, + __global int* featureOffsets, + __global float* oclbuffDetectionResultfeatureVectors, + __global float* oclbuffDetectionResultPosteriors, + __global float* posteriors, + __global int* oclbuffII, + __global float* oclbuffIISqure, + __global float* oclbuffDetectionResultVarious, + const float minVar, + const int numWindows + + ) +{ + __global int *off, *bbox; + __global int * featureVector; + int bbx1,bbx2,bbx3,bbx4; + + + int index,bbox4 ; + float conf=0.0; + int fp0,fp1; + int windowIdx = get_global_id(0); + int gsize = get_global_size(0); + if(windowIdx fp1) + { + index |= 1; + } + off += 2; + } + + featureVector[treeIdx] = index; + + conf = conf+ posteriors[treeIdx * numIndices + index]; + + } + + oclbuffDetectionResultPosteriors[windowIdx] = conf; +// if(conf<0.5 ) +// windowFlags[windowIdx] = 2; +// else +// windowFlags[windowIdx] = 0; + + // if(windowIdx==753666) + // printf("conf=%f,windowFlags[windowIdx]=%d...... \n",conf,windowFlags[windowIdx]); + // if(conf>0.5f) + +Endofkernel: + ; + } +} + +#ifdef DOUBLE_SUPPORT +#ifdef cl_amd_fp64 +#pragma OPENCL EXTENSION cl_amd_fp64:enable +#elif defined (cl_khr_fp64) +#pragma OPENCL EXTENSION cl_khr_fp64:enable +#endif +#endif + +#define LSIZE 256 +#define LSIZE_1 255 +#define LSIZE_2 254 +#define HF_LSIZE 128 +#define LOG_LSIZE 8 +#define LOG_NUM_BANKS 5 +#define NUM_BANKS 32 +#define GET_CONFLICT_OFFSET(lid) ((lid) >> LOG_NUM_BANKS) + + +kernel void integral_cols_DD(__global uchar4 *src,__global int *sum ,__global float *sqsum, + int src_offset,int pre_invalid,int rows,int cols,int src_step,int dst_step) +{ + +// printf("src_offset = %d\n pre_invalid=%d\n,rows=%d\n,cols=%d\n,src_step=%d,dst_step=%d\n",src_offset,pre_invalid,rows,cols,src_step,dst_step); + + int lid = get_local_id(0); + int gid = get_group_id(0); + int4 src_t[2], sum_t[2]; + float4 sqsum_t[2]; + __local int4 lm_sum[2][LSIZE + LOG_LSIZE]; + __local float4 lm_sqsum[2][LSIZE + LOG_LSIZE]; + __local int* sum_p; + __local float* sqsum_p; + src_step = src_step >> 2; + gid = gid << 1; + for(int i = 0; i < rows; i =i + LSIZE_1) + { + src_t[0] = (i + lid < rows ? convert_int4(src[src_offset + (lid+i) * src_step + min(gid, cols - 1)]) : 0); + src_t[1] = (i + lid < rows ? convert_int4(src[src_offset + (lid+i) * src_step + min(gid + 1, cols - 1)]) : 0); + + sum_t[0] = (i == 0 ? 0 : lm_sum[0][LSIZE_2 + LOG_LSIZE]); + sqsum_t[0] = (i == 0 ? (float4)0 : lm_sqsum[0][LSIZE_2 + LOG_LSIZE]); + sum_t[1] = (i == 0 ? 0 : lm_sum[1][LSIZE_2 + LOG_LSIZE]); + sqsum_t[1] = (i == 0 ? (float4)0 : lm_sqsum[1][LSIZE_2 + LOG_LSIZE]); + barrier(CLK_LOCAL_MEM_FENCE); + + int bf_loc = lid + GET_CONFLICT_OFFSET(lid); + lm_sum[0][bf_loc] = src_t[0]; + lm_sqsum[0][bf_loc] = convert_float4(src_t[0] * src_t[0]); + + lm_sum[1][bf_loc] = src_t[1]; + lm_sqsum[1][bf_loc] = convert_float4(src_t[1] * src_t[1]); + + int offset = 1; + for(int d = LSIZE >> 1 ; d > 0; d>>=1) + { + barrier(CLK_LOCAL_MEM_FENCE); + int ai = offset * (((lid & 127)<<1) +1) - 1,bi = ai + offset; + ai += GET_CONFLICT_OFFSET(ai); + bi += GET_CONFLICT_OFFSET(bi); + + if((lid & 127) < d) + { + lm_sum[lid >> 7][bi] += lm_sum[lid >> 7][ai]; + lm_sqsum[lid >> 7][bi] += lm_sqsum[lid >> 7][ai]; + } + offset <<= 1; + } + barrier(CLK_LOCAL_MEM_FENCE); + if(lid < 2) + { + lm_sum[lid][LSIZE_2 + LOG_LSIZE] = 0; + lm_sqsum[lid][LSIZE_2 + LOG_LSIZE] = 0; + } + for(int d = 1; d < LSIZE; d <<= 1) + { + barrier(CLK_LOCAL_MEM_FENCE); + offset >>= 1; + int ai = offset * (((lid & 127)<<1) +1) - 1,bi = ai + offset; + ai += GET_CONFLICT_OFFSET(ai); + bi += GET_CONFLICT_OFFSET(bi); + + if((lid & 127) < d) + { + lm_sum[lid >> 7][bi] += lm_sum[lid >> 7][ai]; + lm_sum[lid >> 7][ai] = lm_sum[lid >> 7][bi] - lm_sum[lid >> 7][ai]; + + lm_sqsum[lid >> 7][bi] += lm_sqsum[lid >> 7][ai]; + lm_sqsum[lid >> 7][ai] = lm_sqsum[lid >> 7][bi] - lm_sqsum[lid >> 7][ai]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + int loc_s0 = gid * dst_step + i + lid - 1 - pre_invalid * dst_step / 4, loc_s1 = loc_s0 + dst_step ; + if(lid > 0 && (i+lid) <= rows) + { + lm_sum[0][bf_loc] += sum_t[0]; + lm_sum[1][bf_loc] += sum_t[1]; + lm_sqsum[0][bf_loc] += sqsum_t[0]; + lm_sqsum[1][bf_loc] += sqsum_t[1]; + sum_p = (__local int*)(&(lm_sum[0][bf_loc])); + sqsum_p = (__local float*)(&(lm_sqsum[0][bf_loc])); + for(int k = 0; k < 4; k++) + { + if(gid * 4 + k >= cols + pre_invalid || gid * 4 + k < pre_invalid) continue; + sum[loc_s0 + k * dst_step / 4] = sum_p[k]; + sqsum[loc_s0 + k * dst_step / 4] = sqsum_p[k]; + + } + sum_p = (__local int*)(&(lm_sum[1][bf_loc])); + sqsum_p = (__local float*)(&(lm_sqsum[1][bf_loc])); + for(int k = 0; k < 4; k++) + { + if(gid * 4 + k + 4 >= cols + pre_invalid) break; + sum[loc_s1 + k * dst_step / 4] = sum_p[k]; + sqsum[loc_s1 + k * dst_step / 4] = sqsum_p[k]; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + } + // if(lid==0 && gid==0) + // printf("sum_start=%d,sqsum_start=%f\n",sum[lid],sqsum[lid] ); + +} + + +kernel void integral_rows_DD(__global int4 *srcsum,__global float4 * srcsqsum,__global int *sum , + __global float *sqsum,int rows,int cols,int src_step,int sum_step, + int sqsum_step,int sum_offset,int sqsum_offset) +{ + + int lid = get_local_id(0); + int gid = get_group_id(0); + int4 src_t[2], sum_t[2]; + float4 sqsrc_t[2],sqsum_t[2]; + __local int4 lm_sum[2][LSIZE + LOG_LSIZE]; + __local float4 lm_sqsum[2][LSIZE + LOG_LSIZE]; + __local int *sum_p; + __local float *sqsum_p; + + + // if(lid==0 && gid==0) + // printf("sum_p=%d,sqsum_p=%f\n",srcsum[0].x,srcsqsum[0].x ); + + src_step = src_step >> 4; + for(int i = 0; i < rows; i =i + LSIZE_1) + { + src_t[0] = i + lid < rows ? srcsum[(lid+i) * src_step + gid * 2] : (int4)0; + sqsrc_t[0] = i + lid < rows ? srcsqsum[(lid+i) * src_step + gid * 2] : (float4)0; + src_t[1] = i + lid < rows ? srcsum[(lid+i) * src_step + gid * 2 + 1] : (int4)0; + sqsrc_t[1] = i + lid < rows ? srcsqsum[(lid+i) * src_step + gid * 2 + 1] : (float4)0; + + sum_t[0] = (i == 0 ? 0 : lm_sum[0][LSIZE_2 + LOG_LSIZE]); + sqsum_t[0] = (i == 0 ? (float4)0 : lm_sqsum[0][LSIZE_2 + LOG_LSIZE]); + sum_t[1] = (i == 0 ? 0 : lm_sum[1][LSIZE_2 + LOG_LSIZE]); + sqsum_t[1] = (i == 0 ? (float4)0 : lm_sqsum[1][LSIZE_2 + LOG_LSIZE]); + barrier(CLK_LOCAL_MEM_FENCE); + + int bf_loc = lid + GET_CONFLICT_OFFSET(lid); + lm_sum[0][bf_loc] = src_t[0]; + lm_sqsum[0][bf_loc] = sqsrc_t[0]; + + lm_sum[1][bf_loc] = src_t[1]; + lm_sqsum[1][bf_loc] = sqsrc_t[1]; + + int offset = 1; + for(int d = LSIZE >> 1 ; d > 0; d>>=1) + { + barrier(CLK_LOCAL_MEM_FENCE); + int ai = offset * (((lid & 127)<<1) +1) - 1,bi = ai + offset; + ai += GET_CONFLICT_OFFSET(ai); + bi += GET_CONFLICT_OFFSET(bi); + + if((lid & 127) < d) + { + lm_sum[lid >> 7][bi] += lm_sum[lid >> 7][ai]; + lm_sqsum[lid >> 7][bi] += lm_sqsum[lid >> 7][ai]; + } + offset <<= 1; + } + barrier(CLK_LOCAL_MEM_FENCE); + if(lid < 2) + { + lm_sum[lid][LSIZE_2 + LOG_LSIZE] = 0; + lm_sqsum[lid][LSIZE_2 + LOG_LSIZE] = 0; + } + for(int d = 1; d < LSIZE; d <<= 1) + { + barrier(CLK_LOCAL_MEM_FENCE); + offset >>= 1; + int ai = offset * (((lid & 127)<<1) +1) - 1,bi = ai + offset; + ai += GET_CONFLICT_OFFSET(ai); + bi += GET_CONFLICT_OFFSET(bi); + + if((lid & 127) < d) + { + lm_sum[lid >> 7][bi] += lm_sum[lid >> 7][ai]; + lm_sum[lid >> 7][ai] = lm_sum[lid >> 7][bi] - lm_sum[lid >> 7][ai]; + + lm_sqsum[lid >> 7][bi] += lm_sqsum[lid >> 7][ai]; + lm_sqsum[lid >> 7][ai] = lm_sqsum[lid >> 7][bi] - lm_sqsum[lid >> 7][ai]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); +// if(gid == 0 && (i + lid) <= rows) + // { + // sum[sum_offset + i + lid] = 0; + // sqsum[sqsum_offset + i + lid] = 0; + // } + // if(i + lid == 0) +// { + // int loc0 = gid * 2 * sum_step; + // int loc1 = gid * 2 * sqsum_step; + // for(int k = 1; k <= 8; k++) + // { + // if(gid * 8 + k > cols) break; + // sum[sum_offset + loc0 + k * sum_step / 4] = 0; + // sqsum[sqsum_offset + loc1 + k * sqsum_step / 4] = 0; + // } + // } + //copy from imgproc_integral_sum.cl + // int loc_s0 = sum_offset + gid * 2 * sum_step + i + lid -1 , loc_s1 = loc_s0 + sum_step ; + + + // int loc_s0 = sum_offset + gid * 2 * sum_step + sum_step / 4 + i + lid, loc_s1 = loc_s0 + sum_step ; + //int loc_sq0 = sqsum_offset + gid * 2 * sqsum_step + sqsum_step / 4 + i + lid, loc_sq1 = loc_sq0 + sqsum_step ; + int loc_s0 = sum_offset + gid * 2 * sum_step + i + lid - 1 , loc_s1 = loc_s0 + sum_step ; + int loc_sq0 = sqsum_offset + gid * 2 * sqsum_step + i + lid - 1, loc_sq1 = loc_sq0 + sqsum_step ; + if(lid > 0 && (i+lid) <= rows) + { + lm_sum[0][bf_loc] += sum_t[0]; + lm_sum[1][bf_loc] += sum_t[1]; + lm_sqsum[0][bf_loc] += sqsum_t[0]; + lm_sqsum[1][bf_loc] += sqsum_t[1]; + sum_p = (__local int*)(&(lm_sum[0][bf_loc])); + sqsum_p = (__local float*)(&(lm_sqsum[0][bf_loc])); + for(int k = 0; k < 4; k++) + { + if(gid * 8 + k >= cols) break; + sum[loc_s0 + k * sum_step / 4] = sum_p[k]; + sqsum[loc_sq0 + k * sqsum_step / 4] = sqsum_p[k]; + } + sum_p = (__local int*)(&(lm_sum[1][bf_loc])); + sqsum_p = (__local float*)(&(lm_sqsum[1][bf_loc])); + for(int k = 0; k < 4; k++) + { + if(gid * 8 + 4 + k >= cols) break; + sum[loc_s1 + k * sum_step / 4] = sum_p[k]; + sqsum[loc_sq1 + k * sqsum_step / 4] = sqsum_p[k]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + // if(lid==0 && gid==0) + // printf("sum=%d,sqsum=%f\n",sum[lid],sqsum[lid] ); +} diff --git a/src/libopentld/kernel/cltldOverlapRect_kernel.cpp b/src/libopentld/kernel/cltldOverlapRect_kernel.cpp new file mode 100644 index 00000000..7fd3975d --- /dev/null +++ b/src/libopentld/kernel/cltldOverlapRect_kernel.cpp @@ -0,0 +1,76 @@ + + + +#pragma OPENCL EXTENSION cl_amd_printf : enable + +__kernel void cltldOverlapRect( + const unsigned int bb10, + const unsigned int bb11, + const unsigned int bb12, + const unsigned int bb13, + + __global int* oclbuffWindowsOffset, + __global float* overlap , + + const unsigned int TLD_WINDOW_SIZE + ) +{ + int windowIdx = get_global_id(0); + int min,max,sum1,sum2; + __global int *bb2= oclbuffWindowsOffset + windowIdx * TLD_WINDOW_SIZE; + + if (bb10 > bb2[0] + bb2[2]) + { + overlap[windowIdx] = 0.0; + goto finished; + } + + if (bb11 > bb2[1] + bb2[3]) + { + overlap[windowIdx] = 0.0; + goto finished; + } + + if (bb10 + bb12 < bb2[0]) + { + overlap[windowIdx] = 0.0; + goto finished; + } + + if (bb11 + bb13 < bb2[1]) + { + overlap[windowIdx] = 0.0; + goto finished; + } + else + { + sum1 = bb10 + bb12; + sum2 = bb2[0] + bb2[2]; + min = sum1bb2[0]? bb10:bb2[0]; + int colInt =min-max; + + sum1 = bb11 + bb13; + sum2 = bb2[1] + bb2[3]; + min = sum1bb2[1]? bb11:bb2[1]; + int rowInt =min-max; + + int intersection = colInt * rowInt; + int area1 = bb12 * bb13; + int area2 = bb2[2] * bb2[3]; + overlap[windowIdx] = intersection / (float)(area1 + area2 - intersection); + } + + finished: + //if(overlap[windowIdx]!=0.0) + //if( windowIdx==249404 ) + //{ + // for(int i=0;i<100;i++) + // printf(".....windowIdx =%d\n",windowIdx); + // printf("windowIdx=%d,overlap[xx]=%f\n",windowIdx,overlap[windowIdx]); + //} + } + + + \ No newline at end of file diff --git a/src/libopentld/kernel/varianceFilter_kernel.cpp b/src/libopentld/kernel/varianceFilter_kernel.cpp new file mode 100644 index 00000000..62230e2c --- /dev/null +++ b/src/libopentld/kernel/varianceFilter_kernel.cpp @@ -0,0 +1,46 @@ + + +#pragma OPENCL EXTENSION cl_amd_printf : enable + +__kernel void varianceFilter( + __global int* oclbuffWindowsOffset, + __global int* oclbuffII, + __global float* oclbuffIISqure, + __global float* oclbuffDetectionResultVarious, +__global float* oclbuffDetectionResultPosteriors, +const unsigned int numWindows, +const float minVar, +__global int* windowFlags) +{ + int windowIdx = get_global_id(0); + //__global int *off = &(oclbuffWindowsOffset[windowIdx*6]) ; + //__global int *off = oclbuffWindowsOffset; + float bboxvar=0.0; + float mX; + float mX2; + + + int off0 =oclbuffWindowsOffset[ windowIdx*6 + 0 ]; + int off1 =oclbuffWindowsOffset[ windowIdx*6 + 1 ]; + int off2 =oclbuffWindowsOffset[ windowIdx*6 + 2 ]; + int off3 =oclbuffWindowsOffset[ windowIdx*6 + 3 ]; + int off4 =oclbuffWindowsOffset[ windowIdx*6 + 4 ]; + int off5 =oclbuffWindowsOffset[ windowIdx*6 + 5 ]; + mX = ( oclbuffII[off3] - oclbuffII[off2] -oclbuffII[off1] + oclbuffII[off0] )/(float) off5; + mX2 = ( oclbuffIISqure[off3] - oclbuffIISqure[off2] -oclbuffIISqure[off1] + oclbuffIISqure[off0] )/(float) off5; + + bboxvar = mX2-mX*mX; + oclbuffDetectionResultVarious[windowIdx] = bboxvar; + + if(bboxvar < minVar) + { + oclbuffDetectionResultPosteriors[windowIdx] = 0.0; + windowFlags[windowIdx] = 1; + } + //windowFlags[windowIdx] = 100; + + +} + + + \ No newline at end of file From 79deba33219013e4e15e589871191e924851e8df Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Mon, 11 Sep 2017 14:14:29 +0800 Subject: [PATCH 05/14] Add one array with numWindows size to store results of the first two classifiers Add one array with numWindows size to store results of the first two classifiers. windowFlags = new float[numWindows]; windowFlags[i] = 0.0f; --- src/libopentld/mftracker/BBPredict.cpp | 5 ++--- src/libopentld/tld/DetectionResult.cpp | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libopentld/mftracker/BBPredict.cpp b/src/libopentld/mftracker/BBPredict.cpp index 9a2556e4..1cac725d 100644 --- a/src/libopentld/mftracker/BBPredict.cpp +++ b/src/libopentld/mftracker/BBPredict.cpp @@ -28,7 +28,6 @@ #include "BBPredict.h" #include -#include #include "Median.h" @@ -38,7 +37,7 @@ */ float getBbWidth(float *bb) { - return std::abs(bb[2] - bb[0] + 1); + return abs(bb[2] - bb[0] + 1); } /** * Returns hight of Boundingbox. @@ -46,7 +45,7 @@ float getBbWidth(float *bb) */ float getBbHeight(float *bb) { - return std::abs(bb[3] - bb[1] + 1); + return abs(bb[3] - bb[1] + 1); } /** * Calculates the new (moved and resized) Bounding box. diff --git a/src/libopentld/tld/DetectionResult.cpp b/src/libopentld/tld/DetectionResult.cpp index 1600d071..6f894a20 100644 --- a/src/libopentld/tld/DetectionResult.cpp +++ b/src/libopentld/tld/DetectionResult.cpp @@ -59,6 +59,9 @@ void DetectionResult::init(int numWindows, int numTrees) variances = new float[numWindows]; posteriors = new float[numWindows]; featureVectors = new int[numWindows * numTrees]; + windowFlags = new float[numWindows]; + for (int i = 0; i < numWindows; i++) + windowFlags[i] = 0.0f; delete confidentIndices; confidentIndices = new vector(); From 0c3a0d693712880e93565ce429e167c538e53391 Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Mon, 11 Sep 2017 15:18:37 +0800 Subject: [PATCH 06/14] Add OpenCL initialization and completed the OpenCL framework based on original framework. Add OpenCL initialization and completed the OpenCL framework based on original framework --- src/libopentld/tld/DetectionResult.h | 1 + src/libopentld/tld/DetectorCascade.cpp | 801 ++++++----- src/libopentld/tld/DetectorCascade.h | 161 ++- src/libopentld/tld/EnsembleClassifier.cpp | 881 ++++++++++--- src/libopentld/tld/EnsembleClassifier.h | 145 +- src/libopentld/tld/IntegralImage.h | 169 ++- src/libopentld/tld/NNClassifier.cpp | 230 ++-- src/libopentld/tld/TLD.cpp | 1466 +++++++++++++-------- src/libopentld/tld/TLD.h | 89 +- src/libopentld/tld/TLDUtil.cpp | 484 ++++--- src/libopentld/tld/TLDUtil.h | 171 +-- src/libopentld/tld/VarianceFilter.cpp | 534 +++++++- src/libopentld/tld/VarianceFilter.h | 79 +- src/opentld/OpenTLD.cpp | 106 +- src/opentld/main/Config.cpp | 1058 +++++++-------- src/opentld/main/Gui.cpp | 258 ++-- src/opentld/main/Main.cpp | 511 +++---- src/opentld/main/Main.h | 2 +- 18 files changed, 4489 insertions(+), 2657 deletions(-) diff --git a/src/libopentld/tld/DetectionResult.h b/src/libopentld/tld/DetectionResult.h index 3ac15d1b..f3efb47f 100644 --- a/src/libopentld/tld/DetectionResult.h +++ b/src/libopentld/tld/DetectionResult.h @@ -43,6 +43,7 @@ class DetectionResult std::vector* confidentIndices; int *featureVectors; float *variances; + float *windowFlags; int numClusters; cv::Rect *detectorBB; //Contains a valid result only if numClusters = 1 diff --git a/src/libopentld/tld/DetectorCascade.cpp b/src/libopentld/tld/DetectorCascade.cpp index 16d7dbe1..eaaa869e 100644 --- a/src/libopentld/tld/DetectorCascade.cpp +++ b/src/libopentld/tld/DetectorCascade.cpp @@ -18,11 +18,11 @@ */ /* - * DetectorCascade.cpp - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* DetectorCascade.cpp +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #include "DetectorCascade.h" @@ -30,315 +30,508 @@ #include "TLDUtil.h" -using namespace cv; - -namespace tld -{ - -//TODO: Convert this to a function -#define sub2idx(x,y,imgWidthStep) ((int) (floor((x)+0.5) + floor((y)+0.5)*(imgWidthStep))) - -DetectorCascade::DetectorCascade() -{ - objWidth = -1; //MUST be set before calling init - objHeight = -1; //MUST be set before calling init - useShift = 1; - imgHeight = -1; - imgWidth = -1; - - shift = 0.1; - minScale = -10; - maxScale = 10; - minSize = 25; - imgWidthStep = -1; - - numTrees = 10; - numFeatures = 13; - - initialised = false; - - foregroundDetector = new ForegroundDetector(); - varianceFilter = new VarianceFilter(); - ensembleClassifier = new EnsembleClassifier(); - nnClassifier = new NNClassifier(); - clustering = new Clustering(); - - detectionResult = new DetectionResult(); -} - -DetectorCascade::~DetectorCascade() -{ - release(); - - delete foregroundDetector; - delete varianceFilter; - delete ensembleClassifier; - delete nnClassifier; - delete detectionResult; - delete clustering; -} - -void DetectorCascade::init() -{ - if(imgWidth == -1 || imgHeight == -1 || imgWidthStep == -1 || objWidth == -1 || objHeight == -1) - { - //printf("Error: Window dimensions not set\n"); //TODO: Convert this to exception - } - - initWindowsAndScales(); - initWindowOffsets(); - - propagateMembers(); - - ensembleClassifier->init(); - - initialised = true; -} - -//TODO: This is error-prone. Better give components a reference to DetectorCascade? -void DetectorCascade::propagateMembers() -{ - detectionResult->init(numWindows, numTrees); - - varianceFilter->windowOffsets = windowOffsets; - ensembleClassifier->windowOffsets = windowOffsets; - ensembleClassifier->imgWidthStep = imgWidthStep; - ensembleClassifier->numScales = numScales; - ensembleClassifier->scales = scales; - ensembleClassifier->numFeatures = numFeatures; - ensembleClassifier->numTrees = numTrees; - nnClassifier->windows = windows; - clustering->windows = windows; - clustering->numWindows = numWindows; - - foregroundDetector->minBlobSize = minSize * minSize; - - foregroundDetector->detectionResult = detectionResult; - varianceFilter->detectionResult = detectionResult; - ensembleClassifier->detectionResult = detectionResult; - nnClassifier->detectionResult = detectionResult; - clustering->detectionResult = detectionResult; -} - -void DetectorCascade::release() -{ - if(!initialised) - { - return; //Do nothing - } - - initialised = false; - - foregroundDetector->release(); - ensembleClassifier->release(); - nnClassifier->release(); - - clustering->release(); - - numWindows = 0; - numScales = 0; - - delete[] scales; - scales = NULL; - delete[] windows; - windows = NULL; - delete[] windowOffsets; - windowOffsets = NULL; - - objWidth = -1; - objHeight = -1; - - detectionResult->release(); -} - -void DetectorCascade::cleanPreviousData() -{ - detectionResult->reset(); -} - -/* returns number of bounding boxes, bounding boxes, number of scales, scales - * bounding boxes are stored in an array of size 5*numBBs using the format - * scales are stored using the format - * - */ -void DetectorCascade::initWindowsAndScales() -{ - - int scanAreaX = 1; // It is important to start with 1/1, because the integral images aren't defined at pos(-1,-1) due to speed reasons - int scanAreaY = 1; - int scanAreaW = imgWidth - 1; - int scanAreaH = imgHeight - 1; +#include - int windowIndex = 0; - scales = new Size[maxScale - minScale + 1]; - numWindows = 0; +#include +#include +#include +#include +#define SUCCESS 0 +#define FAILURE 1 +using namespace std; - int scaleIndex = 0; - - for(int i = minScale; i <= maxScale; i++) - { - float scale = pow(1.2, i); - int w = (int)objWidth * scale; - int h = (int)objHeight * scale; - int ssw, ssh; - - if(useShift) - { - ssw = max(1, w * shift); - ssh = max(1, h * shift); - } - else - { - ssw = 1; - ssh = 1; - } - - if(w < minSize || h < minSize || w > scanAreaW || h > scanAreaH) continue; - - scales[scaleIndex].width = w; - scales[scaleIndex].height = h; - - scaleIndex++; - - numWindows += floor((float)(scanAreaW - w + ssw) / ssw) * floor((float)(scanAreaH - h + ssh) / ssh); - } - - numScales = scaleIndex; - - windows = new int[TLD_WINDOW_SIZE * numWindows]; - - for(scaleIndex = 0; scaleIndex < numScales; scaleIndex++) - { - int w = scales[scaleIndex].width; - int h = scales[scaleIndex].height; - - int ssw, ssh; - - if(useShift) - { - ssw = max(1, w * shift); - ssh = max(1, h * shift); - } - else - { - ssw = 1; - ssh = 1; - } - - for(int y = scanAreaY; y + h <= scanAreaY + scanAreaH; y += ssh) - { - for(int x = scanAreaX; x + w <= scanAreaX + scanAreaW; x += ssw) - { - int *bb = &windows[TLD_WINDOW_SIZE * windowIndex]; - tldCopyBoundaryToArray(x, y, w, h, bb); - bb[4] = scaleIndex; - - windowIndex++; - } - } - - } - - assert(windowIndex == numWindows); -} - -//Creates offsets that can be added to bounding boxes -//offsets are contained in the form delta11, delta12,... (combined index of dw and dh) -//Order: scale->tree->feature -void DetectorCascade::initWindowOffsets() -{ - - windowOffsets = new int[TLD_WINDOW_OFFSET_SIZE * numWindows]; - int *off = windowOffsets; - - int windowSize = TLD_WINDOW_SIZE; - - for(int i = 0; i < numWindows; i++) - { - - int *window = windows + windowSize * i; - *off++ = sub2idx(window[0] - 1, window[1] - 1, imgWidthStep); // x1-1,y1-1 - *off++ = sub2idx(window[0] - 1, window[1] + window[3] - 1, imgWidthStep); // x1-1,y2 - *off++ = sub2idx(window[0] + window[2] - 1, window[1] - 1, imgWidthStep); // x2,y1-1 - *off++ = sub2idx(window[0] + window[2] - 1, window[1] + window[3] - 1, imgWidthStep); // x2,y2 - *off++ = window[4] * 2 * numFeatures * numTrees; // pointer to features for this scale - *off++ = window[2] * window[3]; //Area of bounding box - } -} +using namespace cv; -void DetectorCascade::detect(const Mat &img) +namespace tld { - //For every bounding box, the output is confidence, pattern, variance - - detectionResult->reset(); - - if(!initialised) - { - return; - } - - //Prepare components - foregroundDetector->nextIteration(img); //Calculates foreground - varianceFilter->nextIteration(img); //Calculates integral images - ensembleClassifier->nextIteration(img); - - #pragma omp parallel for - - for(int i = 0; i < numWindows; i++) - { - - int *window = &windows[TLD_WINDOW_SIZE * i]; - if(foregroundDetector->isActive()) - { - bool isInside = false; - - for(size_t j = 0; j < detectionResult->fgList->size(); j++) - { - - int bgBox[4]; - tldRectToArray(detectionResult->fgList->at(j), bgBox); - - if(tldIsInside(window, bgBox)) //TODO: This is inefficient and should be replaced by a quadtree - { - isInside = true; - } - } - - if(!isInside) - { - detectionResult->posteriors[i] = 0; - continue; - } - } - - if(!varianceFilter->filter(i)) - { - detectionResult->posteriors[i] = 0; - continue; - } - - if(!ensembleClassifier->filter(i)) - { - continue; - } - - if(!nnClassifier->filter(img, i)) - { - continue; - } - - detectionResult->confidentIndices->push_back(i); + //TODO: Convert this to a function +#define sub2idx(x,y,imgWidthStep) ((int) (floor((x)+0.5) + floor((y)+0.5)*(imgWidthStep))) + DetectorCascade::DetectorCascade() + { + objWidth = -1; //MUST be set before calling init + objHeight = -1; //MUST be set before calling init + useShift = 1; + imgHeight = -1; + imgWidth = -1; + + shift = 0.1; + minScale = -10; + maxScale = 10; + minSize = 25; + imgWidthStep = -1; + + numTrees = 10; + numFeatures = 13; + numFrame = 0; + + initialised = false; + oclSetup(); + char * kernelName = "..\\..\\..\\src\\libopentld\\kernel\\EnsembleClassifierFilter_kernel.cl.cpp"; + oclBuildKernel(kernelName); + + + foregroundDetector = new ForegroundDetector(); + varianceFilter = new VarianceFilter(); + ensembleClassifier = new EnsembleClassifier(); + nnClassifier = new NNClassifier(); + clustering = new Clustering(); + + detectionResult = new DetectionResult(); + } + + DetectorCascade::~DetectorCascade() + { + release(); + + delete foregroundDetector; + delete varianceFilter; + delete ensembleClassifier; + delete nnClassifier; + delete detectionResult; + delete clustering; + } + + + int DetectorCascade::convertToString(const char *filename, std::string& s) + { + size_t size; + char* str; + std::fstream f(filename, (std::fstream::in | std::fstream::binary)); + + if (f.is_open()) + { + size_t fileSize; + f.seekg(0, std::fstream::end); + size = fileSize = (size_t)f.tellg(); + f.seekg(0, std::fstream::beg); + str = new char[size + 1]; + if (!str) + { + f.close(); + return 0; + } + + f.read(str, fileSize); + f.close(); + str[size] = '\0'; + s = str; + delete[] str; + return 0; + } + cout << "Error: failed to open file\n:" << filename << endl; + return false; + } + + void DetectorCascade::init() + { + if (imgWidth == -1 || imgHeight == -1 || imgWidthStep == -1 || objWidth == -1 || objHeight == -1) + { + //printf("Error: Window dimensions not set\n"); //TODO: Convert this to exception + } + + initWindowsAndScales(); + initWindowOffsets(); + + propagateMembers(); + + ensembleClassifier->init(); + + initialised = true; + } + + //TODO: This is error-prone. Better give components a reference to DetectorCascade? + void DetectorCascade::propagateMembers() + { + detectionResult->init(numWindows, numTrees); + + varianceFilter->windowOffsets = windowOffsets; + varianceFilter->numWindows = numWindows; + ensembleClassifier->numWindows = numWindows;//added by xqc + ensembleClassifier->windowOffsets = windowOffsets; + ensembleClassifier->imgWidthStep = imgWidthStep; + ensembleClassifier->numScales = numScales; + ensembleClassifier->scales = scales; + ensembleClassifier->numFeatures = numFeatures; + ensembleClassifier->numTrees = numTrees; + nnClassifier->windows = windows; + clustering->windows = windows; + clustering->numWindows = numWindows; + + foregroundDetector->minBlobSize = minSize * minSize; + + foregroundDetector->detectionResult = detectionResult; + varianceFilter->detectionResult = detectionResult; + ensembleClassifier->detectionResult = detectionResult; + ensembleClassifier->varianceFilter = varianceFilter; + nnClassifier->detectionResult = detectionResult; + clustering->detectionResult = detectionResult; + + + + //ocl + varianceFilter->platform = platform; + varianceFilter->devices = devices; + varianceFilter->context = context; + varianceFilter->commandQueue = commandQueue; + varianceFilter->program = program; + varianceFilter->kernel_intgegral_cols = kernel_intgegral_cols; + + varianceFilter->kernel_intgegral_rows = kernel_intgegral_rows; + + + // + ensembleClassifier->platform = platform; + ensembleClassifier->devices = devices; + ensembleClassifier->context = context; + ensembleClassifier->commandQueue = commandQueue; + ensembleClassifier->program = program; + ensembleClassifier->variance_ensemble_kernel = kernel_ensemble; + ensembleClassifier->kernel_intgegral_cols_en = kernel_intgegral_cols; + + ensembleClassifier->kernel_intgegral_rows_en = kernel_intgegral_rows; + //kernel_variance = clCreateKernel(program, "varianceFilter", NULL); + + + + + } + + void DetectorCascade::release() + { + if (!initialised) + { + return; //Do nothing + } + + initialised = false; + + foregroundDetector->release(); + ensembleClassifier->release(); + nnClassifier->release(); + + clustering->release(); + + numWindows = 0; + numScales = 0; + + delete[] scales; + scales = NULL; + delete[] windows; + windows = NULL; + delete[] windowOffsets; + windowOffsets = NULL; - } + objWidth = -1; + objHeight = -1; - //Cluster - clustering->clusterConfidentIndices(); + detectionResult->release(); + oclRelease(); + } - detectionResult->containsValidData = true; -} + void DetectorCascade::cleanPreviousData() + { + detectionResult->reset(); + } + + /* returns number of bounding boxes, bounding boxes, number of scales, scales + * bounding boxes are stored in an array of size 5*numBBs using the format + * scales are stored using the format + * + */ + void DetectorCascade::initWindowsAndScales() + { + + int scanAreaX = 1; // It is important to start with 1/1, because the integral images aren't defined at pos(-1,-1) due to speed reasons + int scanAreaY = 1; + int scanAreaW = imgWidth - 1; + int scanAreaH = imgHeight - 1; + + int windowIndex = 0; + + scales = new Size[maxScale - minScale + 1]; + + numWindows = 0; + + int scaleIndex = 0; + + for (int i = minScale; i <= maxScale; i++) + { + float scale = pow(1.2, i); + int w = (int)objWidth * scale; + int h = (int)objHeight * scale; + int ssw, ssh; + + if (useShift) + { + ssw = max(1, w * shift); + ssh = max(1, h * shift); + } + else + { + ssw = 1; + ssh = 1; + } + + if (w < minSize || h < minSize || w > scanAreaW || h > scanAreaH) continue; + + scales[scaleIndex].width = w; + scales[scaleIndex].height = h; + + scaleIndex++; + + numWindows += floor((float)(scanAreaW - w + ssw) / ssw) * floor((float)(scanAreaH - h + ssh) / ssh); + } + + numScales = scaleIndex; + + windows = new int[TLD_WINDOW_SIZE * numWindows]; + + for (scaleIndex = 0; scaleIndex < numScales; scaleIndex++) + { + int w = scales[scaleIndex].width; + int h = scales[scaleIndex].height; + + int ssw, ssh; + + if (useShift) + { + ssw = max(1, w * shift); + ssh = max(1, h * shift); + } + else + { + ssw = 1; + ssh = 1; + } + + for (int y = scanAreaY; y + h <= scanAreaY + scanAreaH; y += ssh) + { + for (int x = scanAreaX; x + w <= scanAreaX + scanAreaW; x += ssw) + { + int *bb = &windows[TLD_WINDOW_SIZE * windowIndex]; + tldCopyBoundaryToArray(x, y, w, h, bb); + bb[4] = scaleIndex; + + windowIndex++; + } + } + + } + + assert(windowIndex == numWindows); + printf("******** DetectorCascade::initWindowsAndScales()********************%d**********************************\n", numWindows); + } + + //oclSetup + int DetectorCascade::oclSetup() + { + /*Step1: Getting platforms and choose an available one.*/ + + platform = NULL; //the chosen platform + status = clGetPlatformIDs(0, NULL, &numPlatforms); + if (status != CL_SUCCESS) + { + cout << "DetectorCascade oclSetup Error: Getting platforms!" << endl; + return 0; + } + + /*For clarity, choose the first available platform. */ + if (numPlatforms > 0) + { + cl_platform_id* platforms = (cl_platform_id*)malloc(numPlatforms* sizeof(cl_platform_id)); + status = clGetPlatformIDs(numPlatforms, platforms, NULL); + platform = platforms[0]; + free(platforms); + } + /*Step 2:Query the platform and choose the first GPU device if has one.Otherwise use the CPU as device.*/ + numDevices = 0; + + status = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 0, NULL, &numDevices); + cout << "numDevices::" << numDevices << endl; + if (numDevices == 0) //no GPU available. + { + cout << "No GPU device available." << endl; + cout << "Choose CPU as default device." << endl; + status = clGetDeviceIDs(platform, CL_DEVICE_TYPE_CPU, 0, NULL, &numDevices); + devices = (cl_device_id*)malloc(numDevices * sizeof(cl_device_id)); + status = clGetDeviceIDs(platform, CL_DEVICE_TYPE_CPU, numDevices, devices, NULL); + } + else + { + devices = (cl_device_id*)malloc(numDevices * sizeof(cl_device_id)); + status = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, numDevices, devices, NULL); + } + for (cl_uint i = 0; i < numDevices; ++i) + { + char deviceName[1024]; + status = clGetDeviceInfo(devices[i], CL_DEVICE_NAME, sizeof(deviceName), + deviceName, NULL); + + std::cout << "Device " << i << " : " << deviceName + << " Device ID is " << devices[i] << std::endl; + } + + /*Step 3: Create context.*/ + context = clCreateContext(NULL, 1, &devices[SELECTED_DEVICE_ID], NULL, NULL, NULL); + + /*Step 4: Creating command queue associate with the context.*/ + commandQueue = clCreateCommandQueue(context, devices[SELECTED_DEVICE_ID], 0, NULL); + } + int DetectorCascade::oclBuildKernel(const char *kernelName) + { + /*Step 5: Create program object */ + //const char *filename = "M:\\OpenTLD\\OpenTLD-master\\OpenTLD-master\\src\\opentld\\HelloWorld_Kernel.cl"; + + + string sourceStr; + status = convertToString(kernelName, sourceStr); + const char *source = sourceStr.c_str(); + size_t sourceSize[] = { strlen(source) }; + program = clCreateProgramWithSource(context, 1, &source, sourceSize, NULL); + + /*Step 6: Build program. */ + status = clBuildProgram(program, 1, &devices[SELECTED_DEVICE_ID], NULL, NULL, NULL); + if (status != CL_SUCCESS) + { + size_t log_size; + char* program_log; + clGetProgramBuildInfo(program, devices[SELECTED_DEVICE_ID], CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size); + program_log = (char*)malloc(log_size + 1); + clGetProgramBuildInfo(program, devices[SELECTED_DEVICE_ID], CL_PROGRAM_BUILD_LOG, log_size + 1, program_log, NULL); + printf("%s\n", program_log); + free(program_log); + + + cout << "DetectorCascade::oclBuildKernelError: Getting platforms!" << endl; + return false; + } + + kernel_variance = clCreateKernel(program, "varianceFilter", NULL); + kernel_ensemble = clCreateKernel(program, "EnsembleClassifierFilter", NULL); + kernel_intgegral_cols = clCreateKernel(program, "integral_cols_DD", NULL); + kernel_intgegral_rows = clCreateKernel(program, "integral_rows_DD", NULL); + } + + void DetectorCascade::oclRelease() + { + /*Step 12: Clean the resources.*/ + if (kernel != NULL) + status = clReleaseKernel(kernel); //Release kernel. + //status = clReleaseProgram(program); //Release the program object. + //status = clReleaseMemObject(inputBuffer); //Release mem object. + //status = clReleaseMemObject(outputBuffer); + //status = clReleaseCommandQueue(commandQueue); //Release Command queue. + //status = clReleaseContext(context); //Release context. + if (devices != NULL) + { + free(devices); + devices = NULL; + } + } + + //Creates offsets that can be added to bounding boxes + //offsets are contained in the form delta11, delta12,... (combined index of dw and dh) + //Order: scale->tree->feature + void DetectorCascade::initWindowOffsets() + { + + windowOffsets = new int[TLD_WINDOW_OFFSET_SIZE * numWindows]; + + int *off = windowOffsets; + + int windowSize = TLD_WINDOW_SIZE; + //#pragma omp parallel for num_threads(8) + for (int i = 0; i < numWindows; i++) + { + int *window = windows + windowSize * i; + *off++ = sub2idx(window[0] - 1, window[1] - 1, imgWidthStep); // x1-1,y1-1 + *off++ = sub2idx(window[0] - 1, window[1] + window[3] - 1, imgWidthStep); // x1-1,y2 + *off++ = sub2idx(window[0] + window[2] - 1, window[1] - 1, imgWidthStep); // x2,y1-1 + *off++ = sub2idx(window[0] + window[2] - 1, window[1] + window[3] - 1, imgWidthStep); // x2,y2 + *off++ = window[4] * 2 * numFeatures * numTrees; // pointer to features for this scale + *off++ = window[2] * window[3]; //Area of bounding box + } + } + + + void DetectorCascade::cldetect(const Mat &img) + { + //For every bounding box, the output is confidence, pattern, variance + //numFrame++; printf("the num of the frame id is %d,numWindows=%d\n", numFrame, numWindows); + + detectionResult->reset(); + if (!initialised) + { + return; + } + + //Prepare components + foregroundDetector->nextIteration(img); //Calculates foreground + //double tic = cvGetTickCount(); + //varianceFilter->nextIteration(img); //Calculates integral images + //varianceFilter->filter2(); + + //double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + //toc = toc / 1000000; + //printf("*****the integral time used is %f************\n", toc);//float fps = 1 / toc; + ensembleClassifier->nextIteration(img); + + + //varianceFilter->clfilter(img); + ensembleClassifier->clfilter(img); + + //for (int i = 0; i < numWindows; i++) + //{ + //printf(" && detectionResult->windowFlags[%d]=%d\n", i, detectionResult->windowFlags[i]); + /* if (detectionResult->windowFlags[i] == 0) + { + printf("xxxxxxxxxxxxxxx windowIdx = %d\nxxxxxxxxxxxxxxxxxx", i); + Sleep(500); + }*/ + + //} + //for (int i = 0; i < numWindows; i++) + //{ + // //printf("flag=1 windowsidx = %d,flag=%d\n", i, detectionResult->windowFlags[i]); + // if (detectionResult->variances != 0) + // { + // printf("haha ... windowsidx = %d, detectionResult->variances=%f\n", i, detectionResult->variances[i]); + // Sleep(1000); + // } + //} + + //printf("detectionResult[753666]=%f\n",detectionResult->posteriors[753666]); + for (int i = 0, count = 0; i < numWindows; i++) + { + // printf("I am victor %d\n", i); + //if (detectionResult->windowFlags[i] == 0) + if (detectionResult->posteriors[i] >= 0.5f) + { + // printf("windowIDx= %d,count=%d\n", i, count++); + count++; + printf("Framenum = %d, I am victor %d,count=%d\n", FramNum, i, count); + if (!nnClassifier->filter(img, i)) + { + continue; + } + //count++; + + detectionResult->confidentIndices->push_back(i); + + } + + } + //printf("confident_size %d\n\n", detectionResult->confidentIndices->size());//zhaodc + + //Cluster + clustering->clusterConfidentIndices(); + + detectionResult->containsValidData = true; + } } /* namespace tld */ diff --git a/src/libopentld/tld/DetectorCascade.h b/src/libopentld/tld/DetectorCascade.h index cbad97e3..cd3f2ea4 100644 --- a/src/libopentld/tld/DetectorCascade.h +++ b/src/libopentld/tld/DetectorCascade.h @@ -17,82 +17,121 @@ * */ /* - * DetectorCascade.h - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* DetectorCascade.h +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #ifndef DETECTORCASCADE_H_ #define DETECTORCASCADE_H_ + #include "DetectionResult.h" #include "ForegroundDetector.h" #include "VarianceFilter.h" #include "EnsembleClassifier.h" #include "Clustering.h" #include "NNClassifier.h" - +#include +#include "switch.h" +#define TRUE 0 +#define FALSE 1 namespace tld { -//Constants -static const int TLD_WINDOW_SIZE = 5; -static const int TLD_WINDOW_OFFSET_SIZE = 6; + //Constants + static const int TLD_WINDOW_SIZE = 5; + static const int TLD_WINDOW_OFFSET_SIZE = 6; -class DetectorCascade -{ - //Working data - int numScales; - cv::Size *scales; -public: - //Configurable members - int minScale; - int maxScale; - bool useShift; - float shift; - int minSize; - int numFeatures; - int numTrees; - - //Needed for init - int imgWidth; - int imgHeight; - int imgWidthStep; - int objWidth; - int objHeight; - - int numWindows; - int *windows; - int *windowOffsets; - - //State data - bool initialised; - - //Components of Detector Cascade - ForegroundDetector *foregroundDetector; - VarianceFilter *varianceFilter; - EnsembleClassifier *ensembleClassifier; - Clustering *clustering; - NNClassifier *nnClassifier; - - DetectionResult *detectionResult; - - void propagateMembers(); - - DetectorCascade(); - ~DetectorCascade(); - - void init(); - - void initWindowOffsets(); - void initWindowsAndScales(); - - void release(); - void cleanPreviousData(); - void detect(const cv::Mat &img); -}; + class DetectorCascade + { + //Working data + int numScales; + cv::Size *scales; + + public: + //Configurable members + int numFrame; + int minScale; + int maxScale; + bool useShift; + float shift; + int minSize; + int numFeatures; + int numTrees; + + //Needed for init + int imgWidth; + int imgHeight; + int imgWidthStep; + int objWidth; + int objHeight; + int FramNum; + int numWindows; + int *windows; + int *windowOffsets; + + + + //******ocl enviroments + + cl_uint numPlatforms; //the NO. of platforms + cl_platform_id platform; //the chosen platform + cl_int status; + cl_uint numDevices; + cl_device_id *devices; + cl_context context; + cl_command_queue commandQueue; + cl_program program; + cl_mem inputBuffer; + cl_mem outputBuffer; + cl_kernel kernel; + + cl_kernel kernel_ensemble; + cl_kernel kernel_variance; + cl_kernel kernel_intgegral_cols; + cl_kernel kernel_intgegral_rows; + + int oclSetup(); + int oclBuildKernel(const char *kernelName); + + + void oclRelease(); + + //******ocl enviroments end + //State data + bool initialised; + + //Components of Detector Cascade + ForegroundDetector *foregroundDetector; + VarianceFilter *varianceFilter; + EnsembleClassifier *ensembleClassifier; + Clustering *clustering; + NNClassifier *nnClassifier; + + DetectionResult *detectionResult; + + + + DetectorCascade(); + ~DetectorCascade(); + + void init(); + + void initWindowOffsets(); + void initWindowsAndScales(); + + void release(); + void cleanPreviousData(); + + void cldetect(const cv::Mat &img); + int convertToString(const char *filename, std::string& s); + + void propagateMembers(); + void detect(const cv::Mat &img); + }; } /* namespace tld */ #endif /* DETECTORCASCADE_H_ */ diff --git a/src/libopentld/tld/EnsembleClassifier.cpp b/src/libopentld/tld/EnsembleClassifier.cpp index 8c43cc05..c9c3591f 100644 --- a/src/libopentld/tld/EnsembleClassifier.cpp +++ b/src/libopentld/tld/EnsembleClassifier.cpp @@ -17,13 +17,14 @@ * */ /* - * EnsembleClassifier.cpp - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* EnsembleClassifier.cpp +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #include "DetectorCascade.h" +#include #include #include @@ -31,221 +32,723 @@ #include #include "EnsembleClassifier.h" - +#include +#include using namespace std; using namespace cv; +#include +#include + +#define ROUND_K(x,k) ((x+(k-1))/(k))*(k) namespace tld { -//TODO: Convert this to a function + //TODO: Convert this to a function #define sub2idx(x,y,widthstep) ((int) (floor((x)+0.5) + floor((y)+0.5)*(widthstep))) -EnsembleClassifier::EnsembleClassifier() : - features(NULL), - featureOffsets(NULL), - posteriors(NULL), - positives(NULL), - negatives(NULL) -{ - numTrees = 10; - numFeatures = 13; - enabled = true; -} + EnsembleClassifier::EnsembleClassifier() : + features(NULL), + featureOffsets(NULL), + posteriors(NULL), + positives(NULL), + negatives(NULL) + { + numTrees = 10; + numFeatures = 13; + enabled = true; -EnsembleClassifier::~EnsembleClassifier() -{ - release(); -} + } + EnsembleClassifier::~EnsembleClassifier() + { + release(); + } -void EnsembleClassifier::init() -{ - numIndices = pow(2.0f, numFeatures); - initFeatureLocations(); - initFeatureOffsets(); - initPosteriors(); -} + void EnsembleClassifier::init() + { + numIndices = pow(2.0f, numFeatures); -void EnsembleClassifier::release() -{ - delete[] features; - features = NULL; - delete[] featureOffsets; - featureOffsets = NULL; - delete[] posteriors; - posteriors = NULL; - delete[] positives; - positives = NULL; - delete[] negatives; - negatives = NULL; -} + initFeatureLocations(); + initFeatureOffsets(); + initPosteriors(); + int tld_window_offset_size = TLD_WINDOW_OFFSET_SIZE; -/* - * Generates random measurements in the format - */ -void EnsembleClassifier::initFeatureLocations() -{ - int size = 2 * 2 * numFeatures * numTrees; - features = new float[size]; + //oclbuffWindowsOffset = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (TLD_WINDOW_OFFSET_SIZE * numWindows) * sizeof(int), (void *)windowOffsets, NULL); + //oclbufffeatureOffsets = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (numScales * numTrees * numFeatures * 2) * sizeof(int), (void *)featureOffsets, NULL); + //oclbuffDetectionResultfeatureVectors = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows * numTrees)* sizeof(int), (void *)detectionResult->featureVectors, NULL); + //oclbuffDetectionResultPosteriors = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)detectionResult->posteriors, NULL); + // + //oclbuffPosteriors = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numTrees * numIndices)* sizeof(float), (void *)posteriors, NULL); + //oclbuffDetectionResultVarious = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)detectionResult->variances, NULL); - for(int i = 0; i < size; i++) - { - features[i] = rand() / (float)RAND_MAX; - } -} -//Creates offsets that can be added to bounding boxes -//offsets are contained in the form delta11, delta12,... (combined index of dw and dh) -//Order: scale.tree->feature -void EnsembleClassifier::initFeatureOffsets() -{ - featureOffsets = new int[numScales * numTrees * numFeatures * 2]; - int *off = featureOffsets; - for(int k = 0; k < numScales; k++) - { - Size scale = scales[k]; - for(int i = 0; i < numTrees; i++) - { - for(int j = 0; j < numFeatures; j++) - { - float *currentFeature = features + (4 * numFeatures) * i + 4 * j; - *off++ = sub2idx((scale.width - 1) * currentFeature[0] + 1, (scale.height - 1) * currentFeature[1] + 1, imgWidthStep); //We add +1 because the index of the bounding box points to x-1, y-1 - *off++ = sub2idx((scale.width - 1) * currentFeature[2] + 1, (scale.height - 1) * currentFeature[3] + 1, imgWidthStep); - } - } - } -} + } + + void EnsembleClassifier::release() + { + delete[] features; + features = NULL; + delete[] featureOffsets; + featureOffsets = NULL; + delete[] posteriors; + posteriors = NULL; + delete[] positives; + positives = NULL; + delete[] negatives; + negatives = NULL; + + + + clReleaseKernel(variance_ensemble_kernel); + clReleaseProgram(program); + clReleaseMemObject(oclbuffWindowsOffset); + clReleaseMemObject(oclbufffeatureOffsets); + clReleaseMemObject(oclbuffDetectionResultfeatureVectors); + clReleaseMemObject(oclbuffDetectionResultPosteriors); + clReleaseMemObject(oclbuffPosteriors); + clReleaseMemObject(oclbuffDetectionResultVarious); + + + } + + /* + * Generates random measurements in the format + */ + void EnsembleClassifier::initFeatureLocations() + { + int size = 2 * 2 * numFeatures * numTrees; + + features = new float[size]; + + for (int i = 0; i < size; i++) + { + features[i] = rand() / (float)RAND_MAX; + } + + } + + //Creates offsets that can be added to bounding boxes + //offsets are contained in the form delta11, delta12,... (combined index of dw and dh) + //Order: scale.tree->feature + void EnsembleClassifier::initFeatureOffsets() + { + + featureOffsets = new int[numScales * numTrees * numFeatures * 2]; + int *off = featureOffsets; + long int offsum1, offsum2; + + for (int k = 0; k < numScales; k++) + { + Size scale = scales[k]; + + for (int i = 0; i < numTrees; i++) + { + for (int j = 0; j < numFeatures; j++) + { + + float *currentFeature = features + (4 * numFeatures) * i + 4 * j; + offsum1 = sub2idx((scale.width - 1) * currentFeature[0] + 1, (scale.height - 1) * currentFeature[1] + 1, imgWidthStep); //We add +1 because the index of the bounding box points to x-1, y-1 + *off++ = offsum1; + + offsum2 = sub2idx((scale.width - 1) * currentFeature[2] + 1, (scale.height - 1) * currentFeature[3] + 1, imgWidthStep); + *off++ = offsum2; + } + } + } + } + + + int EnsembleClassifier::convertToString(const char *filename, std::string& s) + { + size_t size; + char* str; + std::fstream f(filename, (std::fstream::in | std::fstream::binary)); + + if (f.is_open()) + { + size_t fileSize; + f.seekg(0, std::fstream::end); + size = fileSize = (size_t)f.tellg(); + f.seekg(0, std::fstream::beg); + str = new char[size + 1]; + if (!str) + { + f.close(); + return 0; + } + + f.read(str, fileSize); + f.close(); + str[size] = '\0'; + s = str; + delete[] str; + return 0; + } + cout << "Error: failed to open file\n:" << filename << endl; + return false; + } + void EnsembleClassifier::initPosteriors() + { + posteriors = new float[numTrees * numIndices]; + positives = new int[numTrees * numIndices]; + negatives = new int[numTrees * numIndices]; + + for (int i = 0; i < numTrees; i++) + { + for (int j = 0; j < numIndices; j++) + { + posteriors[i * numIndices + j] = 0; + positives[i * numIndices + j] = 0; + negatives[i * numIndices + j] = 0; + } + } + } + + + + void EnsembleClassifier::integralImag_extract(const Mat &src, Mat &sum, Mat &sqsum)//void VarianceFilter::integralImag_extract(const Mat &src) + { + + + int i, j; + + cv::Mat t_sum; + //cv::Mat sum; + cv::Mat t_sqsum; + //cv::Mat sqsum; + + int vlen = 4; cl_event events[2]; + int offset = 0; + int pre_invalid = 0; + int vcols = (pre_invalid + src.cols + vlen - 1) / vlen; + + int w = src.cols, h = src.rows; + int depth = src.depth() == CV_8U ? CV_32S : CV_64F; + int type = CV_MAKE_TYPE(depth, 1); + +#ifdef debug + printf("the type is %d\n", depth); +#endif + t_sum.create(src.cols, src.rows, type); + sum.create(h, w, type); + + t_sqsum.create(src.cols, src.rows, CV_32FC1); + sqsum.create(h, w, CV_32FC1); + + int sum_offset = 0; // sum.offset / vlen; + int sqsum_offset = 0; // sqsum.offset / vlen; + + + cl_mem srcdata = clCreateBuffer(context, CL_MEM_READ_ONLY, (src.cols) * (src.rows) * sizeof(uchar), NULL, NULL); + cl_mem tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(int), NULL, NULL); + cl_mem tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(float), NULL, NULL); + cl_mem sumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(int), NULL, NULL); + cl_mem sqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(float), NULL, NULL); + //use host ptr + //cl_mem srcdata = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (src.cols) * (src.rows) * sizeof(uchar), (void *)src.data, NULL); + //cl_mem tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(int), (void *)t_sum.data, NULL); + //cl_mem tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR , (src.cols) * (src.rows) * sizeof(float), (void *)t_sqsum.data, NULL); + //cl_mem sumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(int), (void *)sum.data, NULL); + //cl_mem sqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(float), (void *)sqsum.data, NULL); + + int step; + int src_step, sum_step; + int t_sum_step, sqsum_step; + + //src_step = 1920; + //t_sum_step = 4320; + //sum_step = 7680; + //sqsum_step = 7680; + int rows = src.rows; + int cols = src.cols; + src_step = src.step; + t_sum_step = t_sum.step; + sum_step = sum.step; + sqsum_step = sqsum.step; + + //printf(""); + status = clEnqueueWriteBuffer(commandQueue, srcdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(uchar), src.data, 0, NULL, NULL); + + assert(status == CL_SUCCESS); + //if (status != CL_SUCCESS) + //{ + // cout << "Error: src.data kernel_intgegral_cols_en EnqueueNDRangeKernel!" << endl; + //} + + status = clSetKernelArg(kernel_intgegral_cols_en, 0, sizeof(cl_mem), (void *)&srcdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&src.data)); + status = clSetKernelArg(kernel_intgegral_cols_en, 1, sizeof(cl_mem), (void *)&tsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sum.data)); + status = clSetKernelArg(kernel_intgegral_cols_en, 2, sizeof(cl_mem), (void *)&tsqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sqsum.data)); + status = clSetKernelArg(kernel_intgegral_cols_en, 3, sizeof(cl_int), (void *)&offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&offset)); + status = clSetKernelArg(kernel_intgegral_cols_en, 4, sizeof(cl_int), (void *)&pre_invalid);//args.push_back(make_pair(sizeof(cl_int), (void *)&pre_invalid)); + status = clSetKernelArg(kernel_intgegral_cols_en, 5, sizeof(cl_int), (void *)&rows);//args.push_back(make_pair(sizeof(cl_int), (void *)&src.rows)); + status = clSetKernelArg(kernel_intgegral_cols_en, 6, sizeof(cl_int), (void *)&cols);//args.push_back(make_pair(sizeof(cl_int), (void *)&src.cols)); + status = clSetKernelArg(kernel_intgegral_cols_en, 7, sizeof(cl_int), &src_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&src.step)); + status = clSetKernelArg(kernel_intgegral_cols_en, 8, sizeof(cl_int), &t_sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.step)); + + + + size_t gt[3] = { ((vcols + 1) / 2) * 256, 1, 1 }, lt[3] = { 256, 1, 1 }; + // //openCLExecuteKernel(src.clCxt, &imgproc_integral, "integral_cols", gt, lt, args, -1, depth); + status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_cols_en, 1, NULL, gt, lt, 0, NULL, &events[0]); + status = clWaitForEvents(1, &events[0]); + assert(status == CL_SUCCESS); + // + // status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_cols_en, 1, NULL, gt, lt, 0, NULL, NULL); + // + //if (status != CL_SUCCESS) + //{ + // cout << "Error: kernel_intgegral_cols_en EnqueueNDRangeKernel!" << endl; + //} + // status = clWaitForEvents(1, &events[0]); + + //if (status != CL_SUCCESS) + // { + // printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + // } + + /*status = clEnqueueReadBuffer(commandQueue, sumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueReadBuffer(commandQueue, sqsumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sqsum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueWriteBuffer(commandQueue, sumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueWriteBuffer(commandQueue, sqsumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sqsum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + */ + + //clReleaseMemObject(tsumdata); + //clReleaseMemObject(tsqsumdata); + + + // tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(int), (void *)t_sum.data, NULL); + // tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(float), (void *)t_sqsum.data, NULL); + + + + status = clSetKernelArg(kernel_intgegral_rows_en, 0, sizeof(cl_mem), (void *)&tsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 1, sizeof(cl_mem), (void *)&tsqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sqsum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 2, sizeof(cl_mem), (void *)&sumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&sum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 3, sizeof(cl_mem), (void *)&sqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&sqsum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 4, sizeof(cl_int), (void *)&t_sum.rows);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.rows)); + status = clSetKernelArg(kernel_intgegral_rows_en, 5, sizeof(cl_int), (void *)&t_sum.cols);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.cols)); + status = clSetKernelArg(kernel_intgegral_rows_en, 6, sizeof(cl_int), (void *)&t_sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.step)); + status = clSetKernelArg(kernel_intgegral_rows_en, 7, sizeof(cl_int), (void *)&sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&sum.step)); + status = clSetKernelArg(kernel_intgegral_rows_en, 8, sizeof(cl_int), (void *)&sqsum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&sqsum.step)); + status = clSetKernelArg(kernel_intgegral_rows_en, 9, sizeof(cl_int), (void *)&sum_offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&sum_offset)); + status = clSetKernelArg(kernel_intgegral_rows_en, 10, sizeof(cl_int), (void *)&sqsum_offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&sqsum_offset)); + + size_t gt2[3] = { t_sum.cols * 32, 1, 1 }, lt2[3] = { 256, 1, 1 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_rows_en, 1, NULL, gt2, lt2, 0, NULL, &events[1]);//openCLExecuteKernel(src.clCxt, &imgproc_integral, "integral_rows", gt2, lt2, args, -1, depth); + + + //if (status != CL_SUCCESS) + //{ + // cout << "Error: kernel_intgegral_rows_en_en EnqueueNDRangeKernel!" << endl; + //} + status = clWaitForEvents(1, &events[1]); + assert(status == CL_SUCCESS); + //if (status != CL_SUCCESS) + //{ + // printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + //} + + //convert clMat to mat + status = clEnqueueReadBuffer(commandQueue, sumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), sum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueReadBuffer(commandQueue, sqsumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), sqsum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + + + //psum = (float*) &sqsum.data; + //for (i = 0; i img = (const unsigned char *)img.data; + } + + //Classical fern algorithm + int EnsembleClassifier::calcFernFeature(int windowIdx, int treeIdx) + { + + int index = 0; + int *bbox = windowOffsets + windowIdx * TLD_WINDOW_OFFSET_SIZE; + int *off = featureOffsets + bbox[4] + treeIdx * 2 * numFeatures; //bbox[4] is pointer to features for the current scale + + for (int i = 0; i < numFeatures; i++) + { + index <<= 1; + + int fp0 = img[bbox[0] + off[0]]; + int fp1 = img[bbox[0] + off[1]]; + + if (fp0 > fp1) + { + index |= 1; + } + + off += 2; + } + + return index; + } + + void EnsembleClassifier::calcFeatureVector(int windowIdx, int *featureVector) + { + for (int i = 0; i < numTrees; i++) + { + featureVector[i] = calcFernFeature(windowIdx, i); + } + } + + float EnsembleClassifier::calcConfidence(int *featureVector) + { + float conf = 0.0; + int temp = 0; + for (int i = 0; i < numTrees; i++) + { + temp = featureVector[i]; + conf += posteriors[i * numIndices + temp]; + //conf += posteriors[i * numIndices + featureVector[i]]; + } + + return conf; + } + + void EnsembleClassifier::classifyWindow2(int windowIdx) + { + float conf = 0.0; + int *featureVector = detectionResult->featureVectors + numTrees * windowIdx; + calcFeatureVector(windowIdx, featureVector); + conf = calcConfidence(featureVector); + detectionResult->posteriors[windowIdx] = conf; + + } + + bool EnsembleClassifier::filter(int i) + { + if (!enabled) return true; + + classifyWindow(i); -void EnsembleClassifier::initPosteriors() -{ - posteriors = new float[numTrees * numIndices]; - positives = new int[numTrees * numIndices]; - negatives = new int[numTrees * numIndices]; - - for(int i = 0; i < numTrees; i++) - { - for(int j = 0; j < numIndices; j++) - { - posteriors[i * numIndices + j] = 0; - positives[i * numIndices + j] = 0; - negatives[i * numIndices + j] = 0; - } - } -} - -void EnsembleClassifier::nextIteration(const Mat &img) -{ - if(!enabled) return; - - this->img = (const unsigned char *)img.data; -} - -//Classical fern algorithm -int EnsembleClassifier::calcFernFeature(int windowIdx, int treeIdx) -{ + if (detectionResult->posteriors[i] < 0.5) return false; - int index = 0; - int *bbox = windowOffsets + windowIdx * TLD_WINDOW_OFFSET_SIZE; - int *off = featureOffsets + bbox[4] + treeIdx * 2 * numFeatures; //bbox[4] is pointer to features for the current scale + return true; + } + void EnsembleClassifier::classifyWindow(int windowIdx) + { + int index, *off, *bbox; + float conf = 0.0; + int treeIdx = 0; + int temp1, temp2; + int *featureVector = detectionResult->featureVectors + numTrees * windowIdx; + int idx = 0; + for (treeIdx = 0; treeIdx < numTrees; treeIdx++) + { - for(int i = 0; i < numFeatures; i++) - { - index <<= 1; + index = 0; - int fp0 = img[bbox[0] + off[0]]; - int fp1 = img[bbox[0] + off[1]]; - - if(fp0 > fp1) - { - index |= 1; - } - - off += 2; - } - - return index; -} - -void EnsembleClassifier::calcFeatureVector(int windowIdx, int *featureVector) -{ - for(int i = 0; i < numTrees; i++) - { - featureVector[i] = calcFernFeature(windowIdx, i); - } -} + bbox = windowOffsets + windowIdx * TLD_WINDOW_OFFSET_SIZE; + off = featureOffsets + bbox[4] + treeIdx * 2 * numFeatures; //bbox[4] is pointer to features for the current scale -float EnsembleClassifier::calcConfidence(int *featureVector) -{ - float conf = 0.0; - - for(int i = 0; i < numTrees; i++) - { - conf += posteriors[i * numIndices + featureVector[i]]; - } - - return conf; -} - -void EnsembleClassifier::classifyWindow(int windowIdx) -{ - int *featureVector = detectionResult->featureVectors + numTrees * windowIdx; - calcFeatureVector(windowIdx, featureVector); - - detectionResult->posteriors[windowIdx] = calcConfidence(featureVector); -} - -bool EnsembleClassifier::filter(int i) -{ - if(!enabled) return true; - - classifyWindow(i); - - if(detectionResult->posteriors[i] < 0.5) return false; - - return true; -} - -void EnsembleClassifier::updatePosterior(int treeIdx, int idx, int positive, int amount) -{ - int arrayIndex = treeIdx * numIndices + idx; - (positive) ? positives[arrayIndex] += amount : negatives[arrayIndex] += amount; - posteriors[arrayIndex] = ((float) positives[arrayIndex]) / (positives[arrayIndex] + negatives[arrayIndex]) / (float) numTrees; -} - -void EnsembleClassifier::updatePosteriors(int *featureVector, int positive, int amount) -{ - - for(int i = 0; i < numTrees; i++) - { - - int idx = featureVector[i]; - updatePosterior(i, idx, positive, amount); - - } -} - -void EnsembleClassifier::learn(int *boundary, int positive, int *featureVector) -{ - if(!enabled) return; + for (int i = 0; i < numFeatures; i++) + { + index <<= 1; - float conf = calcConfidence(featureVector); + int fp0 = img[bbox[0] + off[0]]; + int fp1 = img[bbox[0] + off[1]]; + + if (fp0 > fp1) + { + index |= 1; + } + + off += 2; + idx++; + //if (windowIdx == 10000) + // printf("idx=%d\t fp0=%d,fp1=%d, bbox[0]=%d,off[0]=%d\t,bbox +off=%d,index=%d ***\n",idx, fp0, fp1, bbox[0], off[0], bbox[0] + off[0],index); + } + + featureVector[treeIdx] = index; + + + conf += posteriors[treeIdx * numIndices + featureVector[treeIdx]]; + //if (windowIdx==753666) + //{ + // //printf("idx=%d\t,conf=%f,posterirors[]=%f\n", windowIdx, conf, posteriors[treeIdx * numIndices + featureVector[treeIdx]]); Sleep(1000); + // printf("windowIdx=%d,featureVector[treeIdx=%d]=%d,conf=%f cpu\n", windowIdx, treeIdx, featureVector[treeIdx],conf); + // printf("treeIdx * numIndices + index=%d,posteriors[ xx] = %f.............\n", treeIdx * numIndices + index, posteriors[treeIdx * numIndices + index]); + // Sleep(1000); + //} + + + } + + detectionResult->posteriors[windowIdx] = conf; + //if (conf >= 0.5) + // { + // printf("idx=%d\t,conf=%f\n", windowIdx, conf); Sleep(1000); + // } + } + + + + bool EnsembleClassifier::clfilter(const Mat &matImg) + { + /******* ******/ + /*Begain Integral */ + /****** *******/ + int i, j; + cv::Mat t_sum; //cv::Mat sum; + cv::Mat t_sqsum;//cv::Mat sqsum; + + int vlen = 4; cl_event events[10]; + int offset = 0; + int pre_invalid = 0; + int vcols = (pre_invalid + matImg.cols + vlen - 1) / vlen; + + int w = matImg.cols, h = matImg.rows; + int depth = matImg.depth() == CV_8U ? CV_32S : CV_64F; + int type = CV_MAKE_TYPE(depth, 1); + + + t_sum.create(matImg.cols, matImg.rows, type); + iSumMat_ensemble.create(h, w, type); + + t_sqsum.create(matImg.cols, matImg.rows, CV_32FC1); + fSqreSumMat_ensemble.create(h, w, CV_32FC1); + + int sum_offset = 0; // sum.offset / vlen; + int sqsum_offset = 0; // sqsum.offset / vlen; + + + cl_mem srcdata = clCreateBuffer(context, CL_MEM_READ_ONLY, (matImg.cols) * (matImg.rows) * sizeof(uchar), NULL, NULL); + cl_mem tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (matImg.cols) * (matImg.rows) * sizeof(int), NULL, NULL); + cl_mem tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (matImg.cols) * (matImg.rows) * sizeof(float), NULL, NULL); + cl_mem sumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (matImg.cols) * (matImg.rows) * sizeof(int), NULL, NULL); + cl_mem sqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (matImg.cols) * (matImg.rows) * sizeof(float), NULL, NULL); + + int step; + int src_step, sum_step; + int t_sum_step, sqsum_step; + + + int rows = matImg.rows; + int cols = matImg.cols; + src_step = matImg.step; + t_sum_step = t_sum.step; + sum_step = iSumMat_ensemble.step; + sqsum_step = fSqreSumMat_ensemble.step; + + //printf(""); + status = clEnqueueWriteBuffer(commandQueue, srcdata, CL_FALSE, 0, (matImg.cols) * (matImg.rows) * sizeof(uchar), matImg.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clSetKernelArg(kernel_intgegral_cols_en, 0, sizeof(cl_mem), (void *)&srcdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&matImg.data)); + status = clSetKernelArg(kernel_intgegral_cols_en, 1, sizeof(cl_mem), (void *)&tsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sum.data)); + status = clSetKernelArg(kernel_intgegral_cols_en, 2, sizeof(cl_mem), (void *)&tsqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sqsum.data)); + status = clSetKernelArg(kernel_intgegral_cols_en, 3, sizeof(cl_int), (void *)&offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&offset)); + status = clSetKernelArg(kernel_intgegral_cols_en, 4, sizeof(cl_int), (void *)&pre_invalid);//args.push_back(make_pair(sizeof(cl_int), (void *)&pre_invalid)); + status = clSetKernelArg(kernel_intgegral_cols_en, 5, sizeof(cl_int), (void *)&rows);//args.push_back(make_pair(sizeof(cl_int), (void *)&matImg.rows)); + status = clSetKernelArg(kernel_intgegral_cols_en, 6, sizeof(cl_int), (void *)&cols);//args.push_back(make_pair(sizeof(cl_int), (void *)&matImg.cols)); + status = clSetKernelArg(kernel_intgegral_cols_en, 7, sizeof(cl_int), &src_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&matImg.step)); + status = clSetKernelArg(kernel_intgegral_cols_en, 8, sizeof(cl_int), &t_sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.step)); + + + size_t gt[3] = { ((vcols + 1) / 2) * 256, 1, 1 }, lt[3] = { 256, 1, 1 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_cols_en, 1, NULL, gt, lt, 0, NULL, &events[0]); + status = clWaitForEvents(1, &events[0]); + assert(status == CL_SUCCESS); + + + status = clSetKernelArg(kernel_intgegral_rows_en, 0, sizeof(cl_mem), (void *)&tsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 1, sizeof(cl_mem), (void *)&tsqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sqsum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 2, sizeof(cl_mem), (void *)&sumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&sum.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 3, sizeof(cl_mem), (void *)&sqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&fSqreSumMat_ensemble.data)); + status = clSetKernelArg(kernel_intgegral_rows_en, 4, sizeof(cl_int), (void *)&t_sum.rows);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.rows)); + status = clSetKernelArg(kernel_intgegral_rows_en, 5, sizeof(cl_int), (void *)&t_sum.cols);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.cols)); + status = clSetKernelArg(kernel_intgegral_rows_en, 6, sizeof(cl_int), (void *)&t_sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.step)); + status = clSetKernelArg(kernel_intgegral_rows_en, 7, sizeof(cl_int), (void *)&sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&sum.step)); + status = clSetKernelArg(kernel_intgegral_rows_en, 8, sizeof(cl_int), (void *)&sqsum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&fSqreSumMat_ensemble.step)); + status = clSetKernelArg(kernel_intgegral_rows_en, 9, sizeof(cl_int), (void *)&sum_offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&sum_offset)); + status = clSetKernelArg(kernel_intgegral_rows_en, 10, sizeof(cl_int), (void *)&sqsum_offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&sqsum_offset)); + + size_t gt2[3] = { t_sum.cols * 32, 1, 1 }, lt2[3] = { 256, 1, 1 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_rows_en, 1, NULL, gt2, lt2, 0, NULL, &events[1]);//openCLExecuteKernel(src.clCxt, &imgproc_integral, "integral_rows", gt2, lt2, args, -1, depth); + //if (status != CL_SUCCESS) + //{ + // cout << "Error: kernel_intgegral_rows_en_en EnqueueNDRangeKernel!" << endl; + //} + status = clWaitForEvents(1, &events[1]); + assert(status == CL_SUCCESS); + t_sum.release(); + t_sqsum.release(); + clReleaseMemObject(tsumdata); + clReleaseMemObject(tsqsumdata); + /******* ******/ + /*End Integral */ + /****** *******/ + + + if (!enabled) return true; + int tld_window_offset_size = TLD_WINDOW_OFFSET_SIZE; + + oclbuffWindowsOffset = clCreateBuffer(context, CL_MEM_READ_ONLY, (TLD_WINDOW_OFFSET_SIZE * numWindows) * sizeof(int), NULL, NULL); + oclbufffeatureOffsets = clCreateBuffer(context, CL_MEM_READ_ONLY, (numScales * numTrees * numFeatures * 2) * sizeof(int), NULL, NULL); + oclbuffDetectionResultfeatureVectors = clCreateBuffer(context, CL_MEM_READ_WRITE, (numWindows * numTrees)* sizeof(int), NULL, NULL); + oclbuffDetectionResultPosteriors = clCreateBuffer(context, CL_MEM_READ_WRITE, (numWindows)* sizeof(float), NULL, NULL); + oclbuffPosteriors = clCreateBuffer(context, CL_MEM_READ_WRITE, (numTrees * numIndices)* sizeof(float), NULL, NULL); + oclbuffDetectionResultVarious = clCreateBuffer(context, CL_MEM_READ_WRITE, (numWindows)* sizeof(float), NULL, NULL); + + + status = clEnqueueWriteBuffer(commandQueue, oclbuffWindowsOffset, CL_FALSE, 0, (TLD_WINDOW_OFFSET_SIZE * numWindows) * sizeof(int), (void *)windowOffsets, 0, NULL, NULL); + status = clEnqueueWriteBuffer(commandQueue, oclbufffeatureOffsets, CL_FALSE, 0, (numScales * numTrees * numFeatures * 2) * sizeof(int), (void *)featureOffsets, 0, NULL, NULL); + status = clEnqueueWriteBuffer(commandQueue, oclbuffDetectionResultfeatureVectors, CL_FALSE, 0, (numWindows * numTrees)* sizeof(int), (void *)(void *)detectionResult->featureVectors, 0, NULL, NULL); + status = clEnqueueWriteBuffer(commandQueue, oclbuffDetectionResultPosteriors, CL_FALSE, 0, (numWindows)* sizeof(float), (void *)detectionResult->posteriors, 0, NULL, NULL); + status = clEnqueueWriteBuffer(commandQueue, oclbuffPosteriors, CL_FALSE, 0, (numTrees * numIndices)* sizeof(float), (void *)posteriors, 0, NULL, NULL); + status = clEnqueueWriteBuffer(commandQueue, oclbuffDetectionResultVarious, CL_FALSE, 0, (numWindows)* sizeof(float), (void *)detectionResult->variances, 0, NULL, NULL); + + + status = clSetKernelArg(variance_ensemble_kernel, 0, sizeof(int), (void *)&numIndices); + status = clSetKernelArg(variance_ensemble_kernel, 1, sizeof(int), (void *)&numTrees); + status = clSetKernelArg(variance_ensemble_kernel, 2, sizeof(int), (void *)&numFeatures); + status = clSetKernelArg(variance_ensemble_kernel, 3, sizeof(int), (void *)&tld_window_offset_size); + status = clSetKernelArg(variance_ensemble_kernel, 4, sizeof(cl_mem), (void *)&oclbuffWindowsOffset); + + status = clSetKernelArg(variance_ensemble_kernel, 5, sizeof(cl_mem), (void *)&srcdata); + status = clSetKernelArg(variance_ensemble_kernel, 6, sizeof(cl_mem), (void *)&oclbufffeatureOffsets); + status = clSetKernelArg(variance_ensemble_kernel, 7, sizeof(cl_mem), (void *)&oclbuffDetectionResultfeatureVectors); + status = clSetKernelArg(variance_ensemble_kernel, 8, sizeof(cl_mem), (void *)&oclbuffDetectionResultPosteriors); + status = clSetKernelArg(variance_ensemble_kernel, 9, sizeof(cl_mem), (void *)&oclbuffPosteriors); + + status = clSetKernelArg(variance_ensemble_kernel, 10, sizeof(cl_mem), (void *)&sumdata); + status = clSetKernelArg(variance_ensemble_kernel, 11, sizeof(cl_mem), (void *)&sqsumdata); + status = clSetKernelArg(variance_ensemble_kernel, 12, sizeof(cl_mem), (void *)&oclbuffDetectionResultVarious); + status = clSetKernelArg(variance_ensemble_kernel, 13, sizeof(float), (void *)&varianceFilter->minVar); + status = clSetKernelArg(variance_ensemble_kernel, 14, sizeof(float), (void *)&numWindows); + + + size_t global_work_size[1] = { numWindows }; + // size_t global_work_size[1] = { (numWindows-255)/256*256 }; + //size_t global_work_size[1] = { (numWindows + 16*256 -1 )/4096 * 256 }; 16windows per thread. + size_t local_work_size[1] = { 256 }; + + status = clEnqueueNDRangeKernel(commandQueue, variance_ensemble_kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, &events[2]); + //status = clWaitForEvents(1, &events[2]); + if (status != CL_SUCCESS) + { + + cout << "Error: EnsembleClassifier EnqueueNDRangeKernel!" << endl; + //return false; + } + + status = clEnqueueReadBuffer(commandQueue, oclbuffDetectionResultfeatureVectors, CL_FALSE, 0, (numWindows * numTrees)* sizeof(int), (void *)(void *)detectionResult->featureVectors, 0, NULL, &events[3]); + //status = clWaitForEvents(1, &events[3]); + + status = clEnqueueReadBuffer(commandQueue, oclbuffDetectionResultPosteriors, CL_FALSE, 0, (numWindows)* sizeof(float), (void *)detectionResult->posteriors, 0, NULL, &events[4]); + //status = clWaitForEvents(1, &events[4]); + status = clEnqueueReadBuffer(commandQueue, oclbuffPosteriors, CL_FALSE, 0, (numTrees * numIndices)* sizeof(float), (void *)posteriors, 0, NULL, &events[5]); + //status = clWaitForEvents(1, &events[5]); + status = clEnqueueReadBuffer(commandQueue, oclbuffDetectionResultVarious, CL_FALSE, 0, (numWindows)* sizeof(float), (void *)detectionResult->variances, 0, NULL, &events[6]); + //status = clWaitForEvents(1, &events[6]); + + for (int i = 0; i < 6; i++) + status = clReleaseEvent(events[i]); //cout << "o" << endl; + + + clReleaseMemObject(oclbuffWindowsOffset); + clReleaseMemObject(oclbufffeatureOffsets); + clReleaseMemObject(oclbuffDetectionResultfeatureVectors); + clReleaseMemObject(oclbuffDetectionResultPosteriors); + clReleaseMemObject(oclbuffPosteriors); + clReleaseMemObject(oclbuffDetectionResultVarious); + clReleaseMemObject(srcdata); + clReleaseMemObject(sumdata); + clReleaseMemObject(sqsumdata); + + return true; + } + + void EnsembleClassifier::updatePosterior(int treeIdx, int idx, int positive, int amount) + { + int arrayIndex = treeIdx * numIndices + idx; + (positive) ? positives[arrayIndex] += amount : negatives[arrayIndex] += amount; + posteriors[arrayIndex] = ((float)positives[arrayIndex]) / (positives[arrayIndex] + negatives[arrayIndex]) / (float)numTrees; + } + + void EnsembleClassifier::updatePosteriors(int *featureVector, int positive, int amount) + { + + for (int i = 0; i < numTrees; i++) + { + + int idx = featureVector[i]; + updatePosterior(i, idx, positive, amount); + + } + } - //Update if positive patch and confidence < 0.5 or negative and conf > 0.5 - if((positive && conf < 0.5) || (!positive && conf > 0.5)) - { - updatePosteriors(featureVector, positive, 1); - } + void EnsembleClassifier::learn(int *boundary, int positive, int *featureVector) + { + if (!enabled) return; + + float conf = calcConfidence(featureVector); + + //Update if positive patch and confidence < 0.5 or negative and conf > 0.5 + if ((positive && conf < 0.5) || (!positive && conf > 0.5)) + { + updatePosteriors(featureVector, positive, 1); + } -} + } } /* namespace tld */ diff --git a/src/libopentld/tld/EnsembleClassifier.h b/src/libopentld/tld/EnsembleClassifier.h index 7ca0ffb9..a19eecf6 100644 --- a/src/libopentld/tld/EnsembleClassifier.h +++ b/src/libopentld/tld/EnsembleClassifier.h @@ -17,64 +17,111 @@ * */ /* - * EnsembleClassifier.h - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* EnsembleClassifier.h +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #ifndef ENSEMBLECLASSIFIER_H_ #define ENSEMBLECLASSIFIER_H_ #include +#include namespace tld { -class EnsembleClassifier -{ - const unsigned char *img; - - float calcConfidence(int *featureVector); - int calcFernFeature(int windowIdx, int treeIdx); - void calcFeatureVector(int windowIdx, int *featureVector); - void updatePosteriors(int *featureVector, int positive, int amount); -public: - bool enabled; - - //Configurable members - int numTrees; - int numFeatures; - - int imgWidthStep; - int numScales; - cv::Size *scales; - - int *windowOffsets; - int *featureOffsets; - float *features; - - int numIndices; - - float *posteriors; - int *positives; - int *negatives; - - DetectionResult *detectionResult; - - EnsembleClassifier(); - virtual ~EnsembleClassifier(); - void init(); - void initFeatureLocations(); - void initFeatureOffsets(); - void initPosteriors(); - void release(); - void nextIteration(const cv::Mat &img); - void classifyWindow(int windowIdx); - void updatePosterior(int treeIdx, int idx, int positive, int amount); - void learn(int *boundary, int positive, int *featureVector); - bool filter(int i); -}; + class EnsembleClassifier + { + const unsigned char *img; + + float calcConfidence(int *featureVector); + int calcFernFeature(int windowIdx, int treeIdx); + void calcFeatureVector(int windowIdx, int *featureVector); + void updatePosteriors(int *featureVector, int positive, int amount); + + + + public: + bool enabled; + + //Configurable members + int numTrees; + int numFeatures; + int numWindows; + + int imgWidthStep; + int numScales; + cv::Size *scales; + + int *windowOffsets; + int *featureOffsets; + float *features; + + int numIndices; + + float *posteriors; + int *positives; + int *negatives; + + DetectionResult *detectionResult; + VarianceFilter *varianceFilter; + + cv::Mat iSumMat_ensemble; + cv::Mat fSqreSumMat_ensemble; + + EnsembleClassifier(); + virtual ~EnsembleClassifier(); + void init(); + void initFeatureLocations(); + void initFeatureOffsets(); + void initPosteriors(); + void release(); + void integralImag_extract(const Mat &src, Mat &iSumMat, Mat &fSqreSumMat); + + void nextIteration(const cv::Mat &img); + void classifyWindow(int windowIdx); + void updatePosterior(int treeIdx, int idx, int positive, int amount); + void learn(int *boundary, int positive, int *featureVector); + bool filter(int i); + bool EnsembleClassifier::clfilter(const Mat &img); + void classifyWindow2(int windowIdx); + + + int convertToString(const char *filename, std::string& s); + + + //for opencl begin + + cl_platform_id platform; //the chosen platform + cl_int status; + cl_device_id *devices; + cl_context context; + cl_command_queue commandQueue; + cl_program program; + cl_mem inputBuffer; + cl_mem outputBuffer; + cl_kernel variance_ensemble_kernel; + cl_kernel kernel_intgegral_cols_en, kernel_intgegral_rows_en; + + cl_mem oclbuffWindowsOffset; + cl_mem oclbufffeatureOffsets; + cl_mem oclbuffDetectionResultfeatureVectors; + cl_mem oclbuffDetectionResultPosteriors; + cl_mem oclbuffDetectionwindowFlags; + cl_mem oclbuffPosteriors; + cl_mem oclbuffDetectionResultVarious; + cl_mem oclbuffImgData; + cl_mem oclbuffII; + cl_mem oclbuffIISqure; + size_t local_work_size[1]; + + + + //for opencl end + + }; } /* namespace tld */ #endif /* ENSEMBLECLASSIFIER_H_ */ diff --git a/src/libopentld/tld/IntegralImage.h b/src/libopentld/tld/IntegralImage.h index 6e812148..9de3966a 100644 --- a/src/libopentld/tld/IntegralImage.h +++ b/src/libopentld/tld/IntegralImage.h @@ -18,64 +18,133 @@ */ /* - * IntegralImage.h - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* IntegralImage.h +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #ifndef INTEGRALIMAGE_H_ #define INTEGRALIMAGE_H_ - +#include "switch.h" #include +#include "opencv2/core/core.hpp" +#include "opencv2/highgui/highgui.hpp" +#include "opencv2/ocl/ocl.hpp" +#include "opencv2/nonfree/ocl.hpp" +#include "opencv2/calib3d/calib3d.hpp" +#include "opencv2/nonfree/nonfree.hpp" + +using namespace cv; +using namespace cv::ocl; namespace tld { -template -class IntegralImage -{ -public: - T *data; /* Array containg the entries for the integral image in row-first manner. Of size width*height. Allocated by tldAllocIntImg */ - /* width, height: Dimensions of integral image.*/ - int width; - int height; - - IntegralImage(cv::Size size) - { - data = new T[size.width * size.height]; - } - - virtual ~IntegralImage() - { - delete[] data; - } - - void calcIntImg(const cv::Mat &img, bool squared = false) - { - const unsigned char *input = (const unsigned char *)(img.data); - T *output = data; - - for(int i = 0; i < img.cols; i++) - { - for(int j = 0; j < img.rows; j++) - { - T A = (i > 0) ? output[img.cols * j + i - 1] : 0; - T B = (j > 0) ? output[img.cols * (j - 1) + i] : 0; - T C = (j > 0 && i > 0) ? output[img.cols * (j - 1) + i - 1] : 0; - T value = input[img.step * j + i]; - - if(squared) - { - value = value * value; - } - - output[img.cols * j + i] = A + B - C + value; - } - } - - } -}; + template + class IntegralImage + { + public: + T *data; /* Array containg the entries for the integral image in row-first manner. Of size width*height. Allocated by tldAllocIntImg */ + /* width, height: Dimensions of integral image.*/ + int width; + int height; + + IntegralImage(cv::Size size) + { + data = new T[(size.width) * (size.height)]; + } + + virtual ~IntegralImage() + { + delete[] data; + } + + void oclcalcIntImg(const cv::Mat &img, bool squared = false) + { + // const unsigned char *input = (const unsigned char *)(img.data); + // T *output = data; + int *input; + int *output = data; + DevicesInfo devices; + getOpenCLDevices(devices); + setDevice(devices[SELECTED_DEVICE_ID]); + oclMat img1, img2, MatSqure; + cv::Mat tempMat; + img1 = img; + + + cv::ocl::integral(img1, img2); + tempMat = img2; + + input = (int*)tempMat.data; + //double tic = cvGetTickCount(); + + for (int i = 0; i 0) ? output[img.cols * j + i - 1] : 0; + T B = (j > 0) ? output[img.cols * (j - 1) + i] : 0; + T C = (j > 0 && i > 0) ? output[img.cols * (j - 1) + i - 1] : 0; + T value = input[img.step * j + i]; + + if (squared) + { + value = value * value; + } + + output[img.cols * j + i] = A + B - C + value; + //printf("index, i=%d,j=%d,output value is %d\n", i,j,output[img.cols*j + i]); + } + } + + } + + + }; } /* namespace tld */ diff --git a/src/libopentld/tld/NNClassifier.cpp b/src/libopentld/tld/NNClassifier.cpp index 6eae808a..5c6c313a 100644 --- a/src/libopentld/tld/NNClassifier.cpp +++ b/src/libopentld/tld/NNClassifier.cpp @@ -17,11 +17,11 @@ * */ /* - * NNClassifier.cpp - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* NNClassifier.cpp +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #include "NNClassifier.h" @@ -34,151 +34,155 @@ using namespace cv; namespace tld { -NNClassifier::NNClassifier() -{ - thetaFP = .5; - thetaTP = .65; + NNClassifier::NNClassifier() + { + thetaFP = .5; + thetaTP = .65; - truePositives = new vector(); - falsePositives = new vector(); + truePositives = new vector(); + falsePositives = new vector(); -} + } -NNClassifier::~NNClassifier() -{ - release(); + NNClassifier::~NNClassifier() + { + release(); - delete truePositives; - delete falsePositives; -} + delete truePositives; + delete falsePositives; + } -void NNClassifier::release() -{ - falsePositives->clear(); - truePositives->clear(); -} + void NNClassifier::release() + { + falsePositives->clear(); + truePositives->clear(); + } -float NNClassifier::ncc(float *f1, float *f2) -{ - double corr = 0; - double norm1 = 0; - double norm2 = 0; + float NNClassifier::ncc(float *f1, float *f2) + { + double corr = 0; + double norm1 = 0; + double norm2 = 0; - int size = TLD_PATCH_SIZE * TLD_PATCH_SIZE; + int size = TLD_PATCH_SIZE * TLD_PATCH_SIZE; - for(int i = 0; i < size; i++) - { - corr += f1[i] * f2[i]; - norm1 += f1[i] * f1[i]; - norm2 += f2[i] * f2[i]; - } + for (int i = 0; i < size; i++) + { + corr += f1[i] * f2[i]; + norm1 += f1[i] * f1[i]; + norm2 += f2[i] * f2[i]; + } - // normalization to <0,1> + // normalization to <0,1> - return (corr / sqrt(norm1 * norm2) + 1) / 2.0; -} + return (corr / sqrt(norm1 * norm2) + 1) / 2.0; + } -float NNClassifier::classifyPatch(NormalizedPatch *patch) -{ + float NNClassifier::classifyPatch(NormalizedPatch *patch) + { - if(truePositives->empty()) - { - return 0; - } + if (truePositives->empty()) + { + return 0; + } - if(falsePositives->empty()) - { - return 1; - } + if (falsePositives->empty()) + { + return 1; + } + printf("..................\n"); + printf("truePositives->size()=%d \t,falsePositives->size()=%d\n", truePositives->size(), falsePositives->size()); + printf("..................\n"); + float ccorr_max_p = 0; - float ccorr_max_p = 0; + //Compare patch to positive patches + for (size_t i = 0; i < truePositives->size(); i++) + { + float ccorr = ncc(truePositives->at(i).values, patch->values); - //Compare patch to positive patches - for(size_t i = 0; i < truePositives->size(); i++) - { - float ccorr = ncc(truePositives->at(i).values, patch->values); + if (ccorr > ccorr_max_p) + { + ccorr_max_p = ccorr; + } + } - if(ccorr > ccorr_max_p) - { - ccorr_max_p = ccorr; - } - } + float ccorr_max_n = 0; - float ccorr_max_n = 0; + //Compare patch to negative patches + for (size_t i = 0; i < falsePositives->size(); i++) + { + float ccorr = ncc(falsePositives->at(i).values, patch->values); - //Compare patch to negative patches - for(size_t i = 0; i < falsePositives->size(); i++) - { - float ccorr = ncc(falsePositives->at(i).values, patch->values); + if (ccorr > ccorr_max_n) + { + ccorr_max_n = ccorr; + } + } - if(ccorr > ccorr_max_n) - { - ccorr_max_n = ccorr; - } - } + float dN = 1 - ccorr_max_n; + float dP = 1 - ccorr_max_p; - float dN = 1 - ccorr_max_n; - float dP = 1 - ccorr_max_p; + float distance = dN / (dN + dP); + return distance; + } - float distance = dN / (dN + dP); - return distance; -} + float NNClassifier::classifyBB(const Mat &img, Rect *bb) + { + NormalizedPatch patch; -float NNClassifier::classifyBB(const Mat &img, Rect *bb) -{ - NormalizedPatch patch; + tldExtractNormalizedPatchRect(img, bb, patch.values); + return classifyPatch(&patch); - tldExtractNormalizedPatchRect(img, bb, patch.values); - return classifyPatch(&patch); + } -} + float NNClassifier::classifyWindow(const Mat &img, int windowIdx) + { + NormalizedPatch patch; -float NNClassifier::classifyWindow(const Mat &img, int windowIdx) -{ - NormalizedPatch patch; + int *bbox = &windows[TLD_WINDOW_SIZE * windowIdx]; + tldExtractNormalizedPatchBB(img, bbox, patch.values); - int *bbox = &windows[TLD_WINDOW_SIZE * windowIdx]; - tldExtractNormalizedPatchBB(img, bbox, patch.values); + return classifyPatch(&patch); + } - return classifyPatch(&patch); -} + bool NNClassifier::filter(const Mat &img, int windowIdx) + { + if (!enabled) return true; -bool NNClassifier::filter(const Mat &img, int windowIdx) -{ - if(!enabled) return true; + float conf = classifyWindow(img, windowIdx); + //printf(" windowIdx=%d conf = %f....\n", windowIdx, conf); - float conf = classifyWindow(img, windowIdx); - if(conf < thetaTP) - { - return false; - } + if (conf < thetaTP) + { + return false; + } - return true; -} + return true; + } -void NNClassifier::learn(vector patches) -{ - //TODO: Randomization might be a good idea here - for(size_t i = 0; i < patches.size(); i++) - { + void NNClassifier::learn(vector patches) + { + //TODO: Randomization might be a good idea here + for (size_t i = 0; i < patches.size(); i++) + { - NormalizedPatch patch = patches[i]; + NormalizedPatch patch = patches[i]; - float conf = classifyPatch(&patch); + float conf = classifyPatch(&patch); - if(patch.positive && conf <= thetaTP) - { - truePositives->push_back(patch); - } + if (patch.positive && conf <= thetaTP) + { + truePositives->push_back(patch); + } - if(!patch.positive && conf >= thetaFP) - { - falsePositives->push_back(patch); - } - } + if (!patch.positive && conf >= thetaFP) + { + falsePositives->push_back(patch); + } + } -} + } } /* namespace tld */ diff --git a/src/libopentld/tld/TLD.cpp b/src/libopentld/tld/TLD.cpp index e366a93c..fcaf02f7 100644 --- a/src/libopentld/tld/TLD.cpp +++ b/src/libopentld/tld/TLD.cpp @@ -17,648 +17,956 @@ * */ /* - * TLD.cpp - * - * Created on: Nov 17, 2011 - * Author: Georg Nebehay - */ +* TLD.cpp +* +* Created on: Nov 17, 2011 +* Author: Georg Nebehay +*/ #include "TLD.h" #include - +#include +#include #include "NNClassifier.h" #include "TLDUtil.h" -using namespace std; -using namespace cv; - -namespace tld -{ - -TLD::TLD() -{ - trackerEnabled = true; - detectorEnabled = true; - learningEnabled = true; - alternating = false; - valid = false; - wasValid = false; - learning = false; - currBB = NULL; - prevBB = new Rect(0,0,0,0); - - detectorCascade = new DetectorCascade(); - nnClassifier = detectorCascade->nnClassifier; - - medianFlowTracker = new MedianFlowTracker(); -} - -TLD::~TLD() -{ - storeCurrentData(); - - if(currBB) - { - delete currBB; - currBB = NULL; - } - - if(detectorCascade) - { - delete detectorCascade; - detectorCascade = NULL; - } - - if(medianFlowTracker) - { - delete medianFlowTracker; - medianFlowTracker = NULL; - } - - if(prevBB) - { - delete prevBB; - prevBB = NULL; - } -} - -void TLD::release() -{ - detectorCascade->release(); - medianFlowTracker->cleanPreviousData(); - - if(currBB) - { - delete currBB; - currBB = NULL; - } -} - -void TLD::storeCurrentData() -{ - prevImg.release(); - prevImg = currImg; //Store old image (if any) - if(currBB)//Store old bounding box (if any) - { - prevBB->x = currBB->x; - prevBB->y = currBB->y; - prevBB->width = currBB->width; - prevBB->height = currBB->height; - } - else - { - prevBB->x = 0; - prevBB->y = 0; - prevBB->width = 0; - prevBB->height = 0; - } - - detectorCascade->cleanPreviousData(); //Reset detector results - medianFlowTracker->cleanPreviousData(); - - wasValid = valid; -} - -void TLD::selectObject(const Mat &img, Rect *bb) -{ - //Delete old object - detectorCascade->release(); - - detectorCascade->objWidth = bb->width; - detectorCascade->objHeight = bb->height; - - //Init detector cascade - detectorCascade->init(); - - currImg = img; - if(currBB) - { - delete currBB; - currBB = NULL; - } - currBB = tldCopyRect(bb); - currConf = 1; - valid = true; - - initialLearning(); - -} - -void TLD::processImage(const Mat &img) -{ - storeCurrentData(); - Mat grey_frame; - cvtColor(img, grey_frame, CV_BGR2GRAY); - currImg = grey_frame; // Store new image , right after storeCurrentData(); - - if(trackerEnabled) - { - medianFlowTracker->track(prevImg, currImg, prevBB); - } - - if(detectorEnabled && (!alternating || medianFlowTracker->trackerBB == NULL)) - { - detectorCascade->detect(grey_frame); - } - - fuseHypotheses(); - - learn(); - -} - -void TLD::fuseHypotheses() -{ - Rect *trackerBB = medianFlowTracker->trackerBB; - int numClusters = detectorCascade->detectionResult->numClusters; - Rect *detectorBB = detectorCascade->detectionResult->detectorBB; - - if(currBB) - { - delete currBB; - currBB = NULL; - } - currConf = 0; - valid = false; - - float confDetector = 0; - - if(numClusters == 1) - { - confDetector = nnClassifier->classifyBB(currImg, detectorBB); - } - - if(trackerBB != NULL) - { - float confTracker = nnClassifier->classifyBB(currImg, trackerBB); - if(currBB) - { - delete currBB; - currBB = NULL; - } - - if(numClusters == 1 && confDetector > confTracker && tldOverlapRectRect(*trackerBB, *detectorBB) < 0.5) - { - - currBB = tldCopyRect(detectorBB); - currConf = confDetector; - } - else - { - currBB = tldCopyRect(trackerBB); - currConf = confTracker; - - if(confTracker > nnClassifier->thetaTP) - { - valid = true; - } - else if(wasValid && confTracker > nnClassifier->thetaFP) - { - valid = true; - } - } - } - else if(numClusters == 1) - { - if(currBB) - { - delete currBB; - currBB = NULL; - } - currBB = tldCopyRect(detectorBB); - currConf = confDetector; - } - - /* - float var = CalculateVariance(patch.values, nn->patch_size*nn->patch_size); - - if(var < min_var) { //TODO: Think about incorporating this - printf("%f, %f: Variance too low \n", var, classifier->min_var); - valid = 0; - }*/ -} - -void TLD::initialLearning() -{ - learning = true; //This is just for display purposes - - DetectionResult *detectionResult = detectorCascade->detectionResult; - - detectorCascade->detect(currImg); - - //This is the positive patch - NormalizedPatch patch; - tldExtractNormalizedPatchRect(currImg, currBB, patch.values); - patch.positive = 1; - - float initVar = tldCalcVariance(patch.values, TLD_PATCH_SIZE * TLD_PATCH_SIZE); - detectorCascade->varianceFilter->minVar = initVar / 2; - - - float *overlap = new float[detectorCascade->numWindows]; - tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); - //Add all bounding boxes with high overlap - vector< pair > positiveIndices; - vector negativeIndices; +#include +#include "switch.h" +#include - //First: Find overlapping positive and negative patches - for(int i = 0; i < detectorCascade->numWindows; i++) - { - - if(overlap[i] > 0.6) - { - positiveIndices.push_back(pair(i, overlap[i])); - } - - if(overlap[i] < 0.2) - { - float variance = detectionResult->variances[i]; - - if(!detectorCascade->varianceFilter->enabled || variance > detectorCascade->varianceFilter->minVar) //TODO: This check is unnecessary if minVar would be set before calling detect. - { - negativeIndices.push_back(i); - } - } - } - - sort(positiveIndices.begin(), positiveIndices.end(), tldSortByOverlapDesc); - - vector patches; - - patches.push_back(patch); //Add first patch to patch list - - int numIterations = std::min(positiveIndices.size(), 10); //Take at most 10 bounding boxes (sorted by overlap) - - for(int i = 0; i < numIterations; i++) - { - int idx = positiveIndices.at(i).first; - //Learn this bounding box - //TODO: Somewhere here image warping might be possible - detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], true, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); - } - - srand(1); //TODO: This is not guaranteed to affect random_shuffle - - random_shuffle(negativeIndices.begin(), negativeIndices.end()); - - //Choose 100 random patches for negative examples - for(size_t i = 0; i < std::min(100, negativeIndices.size()); i++) - { - int idx = negativeIndices.at(i); - - NormalizedPatch patch; - tldExtractNormalizedPatchBB(currImg, &detectorCascade->windows[TLD_WINDOW_SIZE * idx], patch.values); - patch.positive = 0; - patches.push_back(patch); - } - - detectorCascade->nnClassifier->learn(patches); - - delete[] overlap; - -} +using namespace std; +using namespace cv; -//Do this when current trajectory is valid -void TLD::learn() +namespace tld { - if(!learningEnabled || !valid || !detectorEnabled) - { - learning = false; - return; - } - learning = true; + TLD::TLD() + { + trackerEnabled = true; + detectorEnabled = true; + learningEnabled = true; + alternating = false; + valid = false; + wasValid = false; + learning = false; + currBB = NULL; + prevBB = new Rect(0, 0, 0, 0); + + detectorCascade = new DetectorCascade(); + nnClassifier = detectorCascade->nnClassifier; + + //TLD tld opencl enviroments + numWindows = detectorCascade->numWindows; + platform = detectorCascade->platform; + devices = detectorCascade->devices; + context = detectorCascade->context; + commandQueue = detectorCascade->commandQueue; + + + + medianFlowTracker = new MedianFlowTracker(); + + int tld_window_size = TLD_WINDOW_SIZE; + char *kernelName = "..\\..\\..\\src\\libopentld\\kernel\\cltldOverlapRect.cl"; + + string sourceStr; + status = convertToString2(kernelName, sourceStr); + const char *source = sourceStr.c_str(); + size_t sourceSize[] = { strlen(source) }; + program = clCreateProgramWithSource(context, 1, &source, sourceSize, NULL); + + /*Step 6: Build program. */ + status = clBuildProgram(program, 1, &devices[SELECTED_DEVICE_ID], NULL, NULL, NULL); + + } + + TLD::~TLD() + { + storeCurrentData(); + + if (currBB) + { + delete currBB; + currBB = NULL; + } + + if (detectorCascade) + { + delete detectorCascade; + detectorCascade = NULL; + } + + if (medianFlowTracker) + { + delete medianFlowTracker; + medianFlowTracker = NULL; + } + + if (prevBB) + { + delete prevBB; + prevBB = NULL; + } + } + + void TLD::release() + { + detectorCascade->release(); + medianFlowTracker->cleanPreviousData(); + + if (currBB) + { + delete currBB; + currBB = NULL; + } + } + int TLD::convertToString2(const char *filename, std::string& s) + { + size_t size; + char* str; + std::fstream f(filename, (std::fstream::in | std::fstream::binary)); + + if (f.is_open()) + { + size_t fileSize; + f.seekg(0, std::fstream::end); + size = fileSize = (size_t)f.tellg(); + f.seekg(0, std::fstream::beg); + str = new char[size + 1]; + if (!str) + { + f.close(); + return 0; + } + + f.read(str, fileSize); + f.close(); + str[size] = '\0'; + s = str; + delete[] str; + return 0; + } + cout << "Error: failed to open file\n:" << filename << endl; + return false; + } + + void TLD::storeCurrentData() + { + prevImg.release(); + prevImg = currImg; //Store old image (if any) + if (currBB)//Store old bounding box (if any) + { + prevBB->x = currBB->x; + prevBB->y = currBB->y; + prevBB->width = currBB->width; + prevBB->height = currBB->height; + } + else + { + prevBB->x = 0; + prevBB->y = 0; + prevBB->width = 0; + prevBB->height = 0; + } + + detectorCascade->cleanPreviousData(); //Reset detector results + medianFlowTracker->cleanPreviousData(); + + wasValid = valid; + } + + void TLD::selectObject(const Mat &img, Rect *bb) + { + +#if opencl + printf("*******************I am using OpenCL***********************\n"); +#else + printf("*******************I am using CPU***********************\n"); +#endif + //Delete old object + detectorCascade->release(); + + detectorCascade->objWidth = bb->width; + detectorCascade->objHeight = bb->height; + + //Init detector cascade + detectorCascade->init(); + + currImg = img; + if (currBB) + { + delete currBB; + currBB = NULL; + } + currBB = tldCopyRect(bb); + currConf = 1; + valid = true; + + initialLearning(); + + } + + void TLD::processImage(const Mat &img) + { + storeCurrentData(); + Mat grey_frame; + + double tic, toc; + cvtColor(img, grey_frame, CV_BGR2GRAY); + currImg = grey_frame; // Store new image , right after storeCurrentData(); + + if (trackerEnabled) + { + medianFlowTracker->track(prevImg, currImg, prevBB); + } + + if (detectorEnabled && (!alternating || medianFlowTracker->trackerBB == NULL)) + { +#if PrintTime_detect + tic = cvGetTickCount(); +#endif +#if clDetect + detectorCascade->cldetect(grey_frame); +#else + detectorCascade->detect(grey_frame); +#endif + +#if PrintTime_detect + toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + printf("*****the tldDect time used is %f************\n", toc);//float fps = 1 / toc; +#endif + + } + +#if PrintTime_fuseHypotheses + tic = cvGetTickCount(); +#endif + + + fuseHypotheses(); +#if PrintTime_fuseHypotheses + toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + printf("*****the PrintTime_fuseHypotheses time used is %f************\n", toc);//float fps = 1 / toc; +#endif +#if PrintTime_learn + tic = cvGetTickCount(); + +#endif + learn(); +#if PrintTime_learn + toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + printf("*****the PrintTime_learn time used is %f************\n", toc);//float fps = 1 / toc; +#endif + + } + + void TLD::fuseHypotheses() + { + Rect *trackerBB = medianFlowTracker->trackerBB; + int numClusters = detectorCascade->detectionResult->numClusters; + Rect *detectorBB = detectorCascade->detectionResult->detectorBB; + + if (currBB) + { + delete currBB; + currBB = NULL; + } + currConf = 0; + valid = false; + + float confDetector = 0; + + if (numClusters == 1) + { + confDetector = nnClassifier->classifyBB(currImg, detectorBB); + } + + if (trackerBB != NULL) + { + float confTracker = nnClassifier->classifyBB(currImg, trackerBB); + if (currBB) + { + delete currBB; + currBB = NULL; + } + + if (numClusters == 1 && confDetector > confTracker && tldOverlapRectRect(*trackerBB, *detectorBB) < 0.5) + { + + currBB = tldCopyRect(detectorBB); + currConf = confDetector; + //printf("FrameNum=%d,currConf = confDetector=%f....\n", detectorCascade->FramNum, currConf); + } + else + { + currBB = tldCopyRect(trackerBB); + currConf = confTracker; + //printf("FrameNum=%d,currConf = confTracker = %f....\n", detectorCascade->FramNum, currConf); + if (confTracker > nnClassifier->thetaTP) + { + valid = true; + } + else if (wasValid && confTracker > nnClassifier->thetaFP) + { + valid = true; + } + } + } + else if (numClusters == 1) + { + if (currBB) + { + delete currBB; + currBB = NULL; + } + currBB = tldCopyRect(detectorBB); + currConf = confDetector; + //printf("FrameNum=%d,numClusters == 1 currConf = confTracker=%f....\n", detectorCascade->FramNum,currConf); + } + + /* + float var = CalculateVariance(patch.values, nn->patch_size*nn->patch_size); + + if(var < min_var) { //TODO: Think about incorporating this + printf("%f, %f: Variance too low \n", var, classifier->min_var); + valid = 0; + }*/ + } + void TLD::cltldOverlapRect_self(int *windows, int numWindows, Rect *boundary, float *overlap) + { + + + cl_event events[1]; + int tld_window_size = TLD_WINDOW_SIZE; + + int bb0, bb1, bb2, bb3; + + bb0 = boundary->x; + bb1 = boundary->y; + bb2 = boundary->width; + bb3 = boundary->height; + //float *overlap = new float[detectorCascade->numWindows]; + + + cl_mem oclbuffWindowsOffset = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (TLD_WINDOW_SIZE * numWindows) * sizeof(int), (void *)windows, NULL); + cl_mem oclbuffOverlap = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)overlap, NULL); + kernel = clCreateKernel(program, "cltldOverlapRect", NULL); + + + status = clSetKernelArg(kernel, 0, sizeof(int), (void *)&bb0); + status = clSetKernelArg(kernel, 1, sizeof(int), (void *)&bb1); + status = clSetKernelArg(kernel, 2, sizeof(int), (void *)&bb2); + status = clSetKernelArg(kernel, 3, sizeof(int), (void *)&bb3); + + status = clSetKernelArg(kernel, 4, sizeof(cl_mem), (void *)&oclbuffWindowsOffset); + status = clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *)&oclbuffOverlap); + status = clSetKernelArg(kernel, 6, sizeof(int), (void *)&tld_window_size); + + + /*Step 10: Running the kernel.*/ + //printf("begore opencl kernel numWindows=%d\n", numWindows); + size_t global_work_size[1] = { numWindows }; + size_t local_work_size[1] = { 256 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, &events[0]); + if (status != CL_SUCCESS) + { + cout << "Error: EnqueueNDRangeKernel!" << endl; + //return false; + } + status = clWaitForEvents(1, &events[0]); + + if (status != CL_SUCCESS) + { + printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + + } + //cout << "o" << endl; + status = clReleaseEvent(events[0]); + + clReleaseMemObject(oclbuffWindowsOffset); + clReleaseMemObject(oclbuffOverlap); + + //return true; + + } + void TLD::cltldOverlapRect(int *windows, int numWindows, Rect *boundary, float *overlap) + { + + //printf("begin using initial clFilter*************************\n"); + cl_event events[1]; + int tld_window_size = TLD_WINDOW_SIZE; + char *kernelName = "..\\..\\..\\src\\libopentld\\kernel\\cltldOverlapRect_kernel.cpp"; + string sourceStr; + status = convertToString2(kernelName, sourceStr); + const char *source = sourceStr.c_str(); + size_t sourceSize[] = { strlen(source) }; + program = clCreateProgramWithSource(context, 1, &source, sourceSize, NULL); + + /*Step 6: Build program. */ + status = clBuildProgram(program, 1, &devices[SELECTED_DEVICE_ID], NULL, NULL, NULL); + //printf("mid using EnsembleClassifier clFilter*************************\n"); + if (status != CL_SUCCESS) + { + cout << "overlap Error: Getting platforms!" << endl; + //return false; + } + int bb0, bb1, bb2, bb3; + + bb0 = boundary->x; + bb1 = boundary->y; + bb2 = boundary->width; + bb3 = boundary->height; + //float *overlap = new float[detectorCascade->numWindows]; + + + //int *windows, int numWindows, Rect *boundary, float *overlap + cl_mem oclbuffWindowsOffset = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (TLD_WINDOW_SIZE * numWindows) * sizeof(int), (void *)windows, NULL); + cl_mem oclbuffOverlap = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)overlap, NULL); + + + /*Step 8: Create kernel object */ + kernel = clCreateKernel(program, "cltldOverlapRect", NULL); + /*Step 9: Sets Kernel arguments.*/ + status = clSetKernelArg(kernel, 0, sizeof(int), (void *)&bb0); + status = clSetKernelArg(kernel, 1, sizeof(int), (void *)&bb1); + status = clSetKernelArg(kernel, 2, sizeof(int), (void *)&bb2); + status = clSetKernelArg(kernel, 3, sizeof(int), (void *)&bb3); + + status = clSetKernelArg(kernel, 4, sizeof(cl_mem), (void *)&oclbuffWindowsOffset); + status = clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *)&oclbuffOverlap); + status = clSetKernelArg(kernel, 6, sizeof(int), (void *)&tld_window_size); + + + /*Step 10: Running the kernel.*/ + //printf("begore opencl kernel numWindows=%d\n", numWindows); + size_t global_work_size[1] = { numWindows }; + size_t local_work_size[1] = { 256 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, &events[0]); + if (status != CL_SUCCESS) + { + cout << "Error: EnqueueNDRangeKernel!" << endl; + //return false; + } + status = clWaitForEvents(1, &events[0]); + + if (status != CL_SUCCESS) + { + printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + + } + //cout << "o" << endl; + status = clReleaseEvent(events[0]); + + clReleaseKernel(kernel); + clReleaseProgram(program); + + clReleaseMemObject(oclbuffWindowsOffset); + clReleaseMemObject(oclbuffOverlap); + + //return true; + + } + + void TLD::initialLearning() + { + learning = true; //This is just for display purposes + + DetectionResult *detectionResult = detectorCascade->detectionResult; +#if clDetect + detectorCascade->cldetect(currImg); +#else + detectorCascade->detect(currImg); +#endif + // + //This is the positive patch + NormalizedPatch patch; + tldExtractNormalizedPatchRect(currImg, currBB, patch.values); + patch.positive = 1; + + float initVar = tldCalcVariance(patch.values, TLD_PATCH_SIZE * TLD_PATCH_SIZE); + detectorCascade->varianceFilter->minVar = initVar / 2; + + + float *overlap = new float[detectorCascade->numWindows]; + // double tic = cvGetTickCount(); +#if cloverlap + cltldOverlapRect_self(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); +#else + tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); +#endif + // double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + // toc = toc / 1000000; + // printf("*****the tldOverlapRect time used is %f************\n", toc);//float fps = 1 / toc; + + // printf("****************i am 248426 overlap[248426]= %f*****************\n", overlap[248426]); + // printf("****************i am 249404 overlap[248426]= %f*****************\n", overlap[249404]); + //Add all bounding boxes with high overlap + + vector< pair > positiveIndices; + vector negativeIndices; + + //First: Find overlapping positive and negative patches + + for (int i = 0; i < detectorCascade->numWindows; i++) + { - DetectionResult *detectionResult = detectorCascade->detectionResult; + if (overlap[i] > 0.6) + { + positiveIndices.push_back(pair(i, overlap[i])); - if(!detectionResult->containsValidData) - { - detectorCascade->detect(currImg); - } + } - //This is the positive patch - NormalizedPatch patch; - tldExtractNormalizedPatchRect(currImg, currBB, patch.values); + if (overlap[i] < 0.2) + { + float variance = detectionResult->variances[i]; + //printf("variance=%f\n", variance); + + if (!detectorCascade->varianceFilter->enabled || variance > detectorCascade->varianceFilter->minVar) //TODO: This check is unnecessary if minVar would be set before calling detect. + { + //printf("variance=%f\n", variance); + negativeIndices.push_back(i); + + } + } + } + cout << "xxxxnegativeIndices.size()xxx" << negativeIndices.size() << endl; + // FILE *fp = fopen("C:\\Users\\kevin\\Desktop\\compResult\\opentld_opencl_overlap.txt", "a+"); + // for (int i = 0; i < negativeIndices.size();i++) + //fprintf(fp, "windowIdx=%d,overlap=%f\n", i, overlap[i]); + // fprintf(fp, "%d\t%f\n", i, negativeIndices.at(i)); + // fclose(fp); + + sort(positiveIndices.begin(), positiveIndices.end(), tldSortByOverlapDesc); + + vector patches; + + patches.push_back(patch); //Add first patch to patch list + + int numIterations = std::min(positiveIndices.size(), 10); //Take at most 10 bounding boxes (sorted by overlap) + + for (int i = 0; i < numIterations; i++) + { + int idx = positiveIndices.at(i).first; + //Learn this bounding box + //TODO: Somewhere here image warping might be possible + detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], true, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); + } + + srand(1); //TODO: This is not guaranteed to affect random_shuffle + + random_shuffle(negativeIndices.begin(), negativeIndices.end()); + + //Choose 100 random patches for negative examples + for (size_t i = 0; i < std::min(100, negativeIndices.size()); i++) + { + int idx = negativeIndices.at(i); + + NormalizedPatch patch; + tldExtractNormalizedPatchBB(currImg, &detectorCascade->windows[TLD_WINDOW_SIZE * idx], patch.values); + patch.positive = 0; + patches.push_back(patch); + } + + detectorCascade->nnClassifier->learn(patches); + + delete[] overlap; + + } - float *overlap = new float[detectorCascade->numWindows]; - tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); + //Do this when current trajectory is valid + void TLD::learn() + { + if (!learningEnabled || !valid || !detectorEnabled) + { + learning = false; + return; + } - //Add all bounding boxes with high overlap + learning = true; - vector > positiveIndices; - vector negativeIndices; - vector negativeIndicesForNN; + DetectionResult *detectionResult = detectorCascade->detectionResult; + + if (!detectionResult->containsValidData) + { + +#if PrintTime_detect + double tic = cvGetTickCount(); +#endif +#if clDetect + detectorCascade->cldetect(currImg); +#else + detectorCascade->detect(currImg); +#endif - //First: Find overlapping positive and negative patches +#if PrintTime_detect + double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + printf("*****the tldDect time used is %f************\n", toc);//float fps = 1 / toc; +#endif + } + + //This is the positive patch + NormalizedPatch patch; + tldExtractNormalizedPatchRect(currImg, currBB, patch.values); + + float *overlap = new float[detectorCascade->numWindows]; + +#if PrintTime_overlap + double tic = cvGetTickCount(); +#endif +#if cloverlap + cltldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); +#else + tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); +#endif +#if PrintTime_overlap + double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + printf("*****the tldOverlapRect time used is %f************\n", toc);//float fps = 1 / toc; +#endif + + //Add all bounding boxes with high overlap + + vector > positiveIndices; + vector negativeIndices; + vector negativeIndicesForNN; + + //First: Find overlapping positive and negative patches + + for (int i = 0; i < detectorCascade->numWindows; i++) + { + + if (overlap[i] > 0.6) + { + positiveIndices.push_back(pair(i, overlap[i])); + //FILE *fp = fopen("opentld_sort_overlap.txt", "a+"); + //fprintf(fp, "windowIdx=%d,overlap=%f\n", i, overlap[i]); + //fclose(fp); + } + + if (overlap[i] < 0.2) + { + if (!detectorCascade->ensembleClassifier->enabled || detectionResult->posteriors[i] > 0.5) //Should be 0.5 according to the paper + { + negativeIndices.push_back(i); + negativeIndicesForNN.push_back(i); + } + + //if(!detectorCascade->ensembleClassifier->enabled || detectionResult->posteriors[i] > 0.5) + //{ + // negativeIndicesForNN.push_back(i); + //} + + } + } + + sort(positiveIndices.begin(), positiveIndices.end(), tldSortByOverlapDesc); + + vector patches; + + patch.positive = 1; + patches.push_back(patch); + //TODO: Flip + + + int numIterations = std::min(positiveIndices.size(), 10); //Take at most 10 bounding boxes (sorted by overlap) + + for (size_t i = 0; i < negativeIndices.size(); i++) + { + int idx = negativeIndices.at(i); + //TODO: Somewhere here image warping might be possible + detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], false, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); + } + + //TODO: Randomization might be a good idea + for (int i = 0; i < numIterations; i++) + { + int idx = positiveIndices.at(i).first; + //TODO: Somewhere here image warping might be possible + detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], true, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); + } + + for (size_t i = 0; i < negativeIndicesForNN.size(); i++) + { + int idx = negativeIndicesForNN.at(i); + + NormalizedPatch patch; + tldExtractNormalizedPatchBB(currImg, &detectorCascade->windows[TLD_WINDOW_SIZE * idx], patch.values); + patch.positive = 0; + patches.push_back(patch); + } + + detectorCascade->nnClassifier->learn(patches); + //cout << "xxxxnegativeIndices.size()xxx ....." << negativeIndices.size() << "negativeIndicesForNN.size()=...;;" << negativeIndicesForNN.size() << endl; + //cout << "NN has now " << detectorCascade->nnClassifier->truePositives->size() << " positives and " << detectorCascade->nnClassifier->falsePositives->size() << " negatives.\n"; + + delete[] overlap; + /*debug float *p; + p = detectorCascade->ensembleClassifier->posteriors; + for (int i = 0; i < 10; i++) + { + for (int j = 0; j < 8192; j++) + { + + if (p[i * 8192 + j] > 0.11) + printf("the tree is %d, the ensembleClassifier posteriors is %f\n",i , p[i * 8192 + j]); + } + printf("xxxxxxxxxxxxxxxxxxxxxx the tree is %d\n", i); + } + */ + } + + typedef struct + { + int index; + int P; + int N; + } TldExportEntry; + + void TLD::writeToFile(const char *path) + { + NNClassifier *nn = detectorCascade->nnClassifier; + EnsembleClassifier *ec = detectorCascade->ensembleClassifier; + + FILE *file = fopen(path, "w"); + fprintf(file, "#Tld ModelExport\n"); + fprintf(file, "%d #width\n", detectorCascade->objWidth); + fprintf(file, "%d #height\n", detectorCascade->objHeight); + fprintf(file, "%f #min_var\n", detectorCascade->varianceFilter->minVar); + fprintf(file, "%d #Positive Sample Size\n", nn->truePositives->size()); + + + + for (size_t s = 0; s < nn->truePositives->size(); s++) + { + float *imageData = nn->truePositives->at(s).values; + + for (int i = 0; i < TLD_PATCH_SIZE; i++) + { + for (int j = 0; j < TLD_PATCH_SIZE; j++) + { + fprintf(file, "%f ", imageData[i * TLD_PATCH_SIZE + j]); + } + + fprintf(file, "\n"); + } + } + + fprintf(file, "%d #Negative Sample Size\n", nn->falsePositives->size()); + + for (size_t s = 0; s < nn->falsePositives->size(); s++) + { + float *imageData = nn->falsePositives->at(s).values; + + for (int i = 0; i < TLD_PATCH_SIZE; i++) + { + for (int j = 0; j < TLD_PATCH_SIZE; j++) + { + fprintf(file, "%f ", imageData[i * TLD_PATCH_SIZE + j]); + } + + fprintf(file, "\n"); + } + } + + fprintf(file, "%d #numtrees\n", ec->numTrees); + detectorCascade->numTrees = ec->numTrees; + fprintf(file, "%d #numFeatures\n", ec->numFeatures); + detectorCascade->numFeatures = ec->numFeatures; + + for (int i = 0; i < ec->numTrees; i++) + { + fprintf(file, "#Tree %d\n", i); + + for (int j = 0; j < ec->numFeatures; j++) + { + float *features = ec->features + 4 * ec->numFeatures * i + 4 * j; + fprintf(file, "%f %f %f %f # Feature %d\n", features[0], features[1], features[2], features[3], j); + } + + //Collect indices + vector list; + + for (int index = 0; index < pow(2.0f, ec->numFeatures); index++) + { + int p = ec->positives[i * ec->numIndices + index]; + + if (p != 0) + { + TldExportEntry entry; + entry.index = index; + entry.P = p; + entry.N = ec->negatives[i * ec->numIndices + index]; + list.push_back(entry); + } + } + + fprintf(file, "%d #numLeaves\n", list.size()); - for(int i = 0; i < detectorCascade->numWindows; i++) - { + for (size_t j = 0; j < list.size(); j++) + { + TldExportEntry entry = list.at(j); + fprintf(file, "%d %d %d\n", entry.index, entry.P, entry.N); + } + } - if(overlap[i] > 0.6) - { - positiveIndices.push_back(pair(i, overlap[i])); - } + fclose(file); - if(overlap[i] < 0.2) - { - if(!detectorCascade->ensembleClassifier->enabled || detectionResult->posteriors[i] > 0.5) //Should be 0.5 according to the paper - { - negativeIndices.push_back(i); - } + } - if(!detectorCascade->ensembleClassifier->enabled || detectionResult->posteriors[i] > 0.5) - { - negativeIndicesForNN.push_back(i); - } - - } - } - - sort(positiveIndices.begin(), positiveIndices.end(), tldSortByOverlapDesc); - - vector patches; - - patch.positive = 1; - patches.push_back(patch); - //TODO: Flip - - - int numIterations = std::min(positiveIndices.size(), 10); //Take at most 10 bounding boxes (sorted by overlap) - - for(size_t i = 0; i < negativeIndices.size(); i++) - { - int idx = negativeIndices.at(i); - //TODO: Somewhere here image warping might be possible - detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], false, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); - } - - //TODO: Randomization might be a good idea - for(int i = 0; i < numIterations; i++) - { - int idx = positiveIndices.at(i).first; - //TODO: Somewhere here image warping might be possible - detectorCascade->ensembleClassifier->learn(&detectorCascade->windows[TLD_WINDOW_SIZE * idx], true, &detectionResult->featureVectors[detectorCascade->numTrees * idx]); - } - - for(size_t i = 0; i < negativeIndicesForNN.size(); i++) - { - int idx = negativeIndicesForNN.at(i); - - NormalizedPatch patch; - tldExtractNormalizedPatchBB(currImg, &detectorCascade->windows[TLD_WINDOW_SIZE * idx], patch.values); - patch.positive = 0; - patches.push_back(patch); - } - - detectorCascade->nnClassifier->learn(patches); - - //cout << "NN has now " << detectorCascade->nnClassifier->truePositives->size() << " positives and " << detectorCascade->nnClassifier->falsePositives->size() << " negatives.\n"; - - delete[] overlap; -} - -typedef struct -{ - int index; - int P; - int N; -} TldExportEntry; - -void TLD::writeToFile(const char *path) -{ - NNClassifier *nn = detectorCascade->nnClassifier; - EnsembleClassifier *ec = detectorCascade->ensembleClassifier; - - FILE *file = fopen(path, "w"); - fprintf(file, "#Tld ModelExport\n"); - fprintf(file, "%d #width\n", detectorCascade->objWidth); - fprintf(file, "%d #height\n", detectorCascade->objHeight); - fprintf(file, "%f #min_var\n", detectorCascade->varianceFilter->minVar); - fprintf(file, "%d #Positive Sample Size\n", nn->truePositives->size()); - - - - for(size_t s = 0; s < nn->truePositives->size(); s++) - { - float *imageData = nn->truePositives->at(s).values; - - for(int i = 0; i < TLD_PATCH_SIZE; i++) - { - for(int j = 0; j < TLD_PATCH_SIZE; j++) - { - fprintf(file, "%f ", imageData[i * TLD_PATCH_SIZE + j]); - } - - fprintf(file, "\n"); - } - } - - fprintf(file, "%d #Negative Sample Size\n", nn->falsePositives->size()); - - for(size_t s = 0; s < nn->falsePositives->size(); s++) - { - float *imageData = nn->falsePositives->at(s).values; - - for(int i = 0; i < TLD_PATCH_SIZE; i++) - { - for(int j = 0; j < TLD_PATCH_SIZE; j++) - { - fprintf(file, "%f ", imageData[i * TLD_PATCH_SIZE + j]); - } - - fprintf(file, "\n"); - } - } - - fprintf(file, "%d #numtrees\n", ec->numTrees); - detectorCascade->numTrees = ec->numTrees; - fprintf(file, "%d #numFeatures\n", ec->numFeatures); - detectorCascade->numFeatures = ec->numFeatures; - - for(int i = 0; i < ec->numTrees; i++) - { - fprintf(file, "#Tree %d\n", i); - - for(int j = 0; j < ec->numFeatures; j++) - { - float *features = ec->features + 4 * ec->numFeatures * i + 4 * j; - fprintf(file, "%f %f %f %f # Feature %d\n", features[0], features[1], features[2], features[3], j); - } - - //Collect indices - vector list; - - for(int index = 0; index < pow(2.0f, ec->numFeatures); index++) - { - int p = ec->positives[i * ec->numIndices + index]; - - if(p != 0) - { - TldExportEntry entry; - entry.index = index; - entry.P = p; - entry.N = ec->negatives[i * ec->numIndices + index]; - list.push_back(entry); - } - } - - fprintf(file, "%d #numLeaves\n", list.size()); - - for(size_t j = 0; j < list.size(); j++) - { - TldExportEntry entry = list.at(j); - fprintf(file, "%d %d %d\n", entry.index, entry.P, entry.N); - } - } - - fclose(file); - -} - -void TLD::readFromFile(const char *path) -{ - release(); + void TLD::readFromFile(const char *path) + { + release(); - NNClassifier *nn = detectorCascade->nnClassifier; - EnsembleClassifier *ec = detectorCascade->ensembleClassifier; + NNClassifier *nn = detectorCascade->nnClassifier; + EnsembleClassifier *ec = detectorCascade->ensembleClassifier; - FILE *file = fopen(path, "r"); + FILE *file = fopen(path, "r"); - if(file == NULL) - { - printf("Error: Model not found: %s\n", path); - exit(1); - } + if (file == NULL) + { + printf("Error: Model not found: %s\n", path); + exit(1); + } - int MAX_LEN = 255; - char str_buf[255]; - fgets(str_buf, MAX_LEN, file); /*Skip line*/ + int MAX_LEN = 255; + char str_buf[255]; + fgets(str_buf, MAX_LEN, file); /*Skip line*/ - fscanf(file, "%d \n", &detectorCascade->objWidth); - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - fscanf(file, "%d \n", &detectorCascade->objHeight); - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + fscanf(file, "%d \n", &detectorCascade->objWidth); + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + fscanf(file, "%d \n", &detectorCascade->objHeight); + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - fscanf(file, "%f \n", &detectorCascade->varianceFilter->minVar); - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + fscanf(file, "%f \n", &detectorCascade->varianceFilter->minVar); + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - int numPositivePatches; - fscanf(file, "%d \n", &numPositivePatches); - fgets(str_buf, MAX_LEN, file); /*Skip line*/ + int numPositivePatches; + fscanf(file, "%d \n", &numPositivePatches); + fgets(str_buf, MAX_LEN, file); /*Skip line*/ - for(int s = 0; s < numPositivePatches; s++) - { - NormalizedPatch patch; + for (int s = 0; s < numPositivePatches; s++) + { + NormalizedPatch patch; - for(int i = 0; i < 15; i++) //Do 15 times - { + for (int i = 0; i < 15; i++) //Do 15 times + { - fgets(str_buf, MAX_LEN, file); /*Read sample*/ + fgets(str_buf, MAX_LEN, file); /*Read sample*/ - char *pch; - pch = strtok(str_buf, " \n"); - int j = 0; + char *pch; + pch = strtok(str_buf, " \n"); + int j = 0; - while(pch != NULL) - { - float val = atof(pch); - patch.values[i * TLD_PATCH_SIZE + j] = val; + while (pch != NULL) + { + float val = atof(pch); + patch.values[i * TLD_PATCH_SIZE + j] = val; - pch = strtok(NULL, " \n"); + pch = strtok(NULL, " \n"); - j++; - } - } + j++; + } + } - nn->truePositives->push_back(patch); - } + nn->truePositives->push_back(patch); + } - int numNegativePatches; - fscanf(file, "%d \n", &numNegativePatches); - fgets(str_buf, MAX_LEN, file); /*Skip line*/ + int numNegativePatches; + fscanf(file, "%d \n", &numNegativePatches); + fgets(str_buf, MAX_LEN, file); /*Skip line*/ - for(int s = 0; s < numNegativePatches; s++) - { - NormalizedPatch patch; + for (int s = 0; s < numNegativePatches; s++) + { + NormalizedPatch patch; - for(int i = 0; i < 15; i++) //Do 15 times - { + for (int i = 0; i < 15; i++) //Do 15 times + { - fgets(str_buf, MAX_LEN, file); /*Read sample*/ + fgets(str_buf, MAX_LEN, file); /*Read sample*/ - char *pch; - pch = strtok(str_buf, " \n"); - int j = 0; + char *pch; + pch = strtok(str_buf, " \n"); + int j = 0; - while(pch != NULL) - { - float val = atof(pch); - patch.values[i * TLD_PATCH_SIZE + j] = val; + while (pch != NULL) + { + float val = atof(pch); + patch.values[i * TLD_PATCH_SIZE + j] = val; - pch = strtok(NULL, " \n"); + pch = strtok(NULL, " \n"); - j++; - } - } + j++; + } + } - nn->falsePositives->push_back(patch); - } + nn->falsePositives->push_back(patch); + } - fscanf(file, "%d \n", &ec->numTrees); - detectorCascade->numTrees = ec->numTrees; - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + fscanf(file, "%d \n", &ec->numTrees); + detectorCascade->numTrees = ec->numTrees; + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - fscanf(file, "%d \n", &ec->numFeatures); - detectorCascade->numFeatures = ec->numFeatures; - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + fscanf(file, "%d \n", &ec->numFeatures); + detectorCascade->numFeatures = ec->numFeatures; + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - int size = 2 * 2 * ec->numFeatures * ec->numTrees; - ec->features = new float[size]; - ec->numIndices = pow(2.0f, ec->numFeatures); - ec->initPosteriors(); + int size = 2 * 2 * ec->numFeatures * ec->numTrees; + ec->features = new float[size]; + ec->numIndices = pow(2.0f, ec->numFeatures); + ec->initPosteriors(); - for(int i = 0; i < ec->numTrees; i++) - { - fgets(str_buf, MAX_LEN, file); /*Skip line*/ + for (int i = 0; i < ec->numTrees; i++) + { + fgets(str_buf, MAX_LEN, file); /*Skip line*/ - for(int j = 0; j < ec->numFeatures; j++) - { - float *features = ec->features + 4 * ec->numFeatures * i + 4 * j; - fscanf(file, "%f %f %f %f", &features[0], &features[1], &features[2], &features[3]); - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - } + for (int j = 0; j < ec->numFeatures; j++) + { + float *features = ec->features + 4 * ec->numFeatures * i + 4 * j; + fscanf(file, "%f %f %f %f", &features[0], &features[1], &features[2], &features[3]); + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + } - /* read number of leaves*/ - int numLeaves; - fscanf(file, "%d \n", &numLeaves); - fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ + /* read number of leaves*/ + int numLeaves; + fscanf(file, "%d \n", &numLeaves); + fgets(str_buf, MAX_LEN, file); /*Skip rest of line*/ - for(int j = 0; j < numLeaves; j++) - { - TldExportEntry entry; - fscanf(file, "%d %d %d \n", &entry.index, &entry.P, &entry.N); - ec->updatePosterior(i, entry.index, 1, entry.P); - ec->updatePosterior(i, entry.index, 0, entry.N); - } - } + for (int j = 0; j < numLeaves; j++) + { + TldExportEntry entry; + fscanf(file, "%d %d %d \n", &entry.index, &entry.P, &entry.N); + ec->updatePosterior(i, entry.index, 1, entry.P); + ec->updatePosterior(i, entry.index, 0, entry.N); + } + } - detectorCascade->initWindowsAndScales(); - detectorCascade->initWindowOffsets(); + detectorCascade->initWindowsAndScales(); + detectorCascade->initWindowOffsets(); - detectorCascade->propagateMembers(); + detectorCascade->propagateMembers(); - detectorCascade->initialised = true; + detectorCascade->initialised = true; - ec->initFeatureOffsets(); + ec->initFeatureOffsets(); - fclose(file); -} + fclose(file); + } } /* namespace tld */ diff --git a/src/libopentld/tld/TLD.h b/src/libopentld/tld/TLD.h index 9723d091..e477d5f5 100644 --- a/src/libopentld/tld/TLD.h +++ b/src/libopentld/tld/TLD.h @@ -18,11 +18,11 @@ */ /* - * TLD.h - * - * Created on: Nov 17, 2011 - * Author: Georg Nebehay - */ +* TLD.h +* +* Created on: Nov 17, 2011 +* Author: Georg Nebehay +*/ #ifndef TLD_H_ #define TLD_H_ @@ -31,42 +31,61 @@ #include "MedianFlowTracker.h" #include "DetectorCascade.h" +#include namespace tld { -class TLD -{ - void storeCurrentData(); - void fuseHypotheses(); - void learn(); - void initialLearning(); -public: - bool trackerEnabled; - bool detectorEnabled; - bool learningEnabled; - bool alternating; + class TLD + { + void storeCurrentData(); + void fuseHypotheses(); + void learn(); + void initialLearning(); + public: + bool trackerEnabled; + bool detectorEnabled; + bool learningEnabled; + bool alternating; + + MedianFlowTracker *medianFlowTracker; + DetectorCascade *detectorCascade; + NNClassifier *nnClassifier; + bool valid; + bool wasValid; + cv::Mat prevImg; + cv::Mat currImg; + cv::Rect *prevBB; + cv::Rect *currBB; + float currConf; + bool learning; + + TLD(); + virtual ~TLD(); + void release(); + void selectObject(const cv::Mat &img, cv::Rect *bb); + void processImage(const cv::Mat &img); + void writeToFile(const char *path); + void readFromFile(const char *path); + - MedianFlowTracker *medianFlowTracker; - DetectorCascade *detectorCascade; - NNClassifier *nnClassifier; - bool valid; - bool wasValid; - cv::Mat prevImg; - cv::Mat currImg; - cv::Rect *prevBB; - cv::Rect *currBB; - float currConf; - bool learning; + //for opencl begin + int numWindows; + cl_platform_id platform; //the chosen platform + cl_int status; + cl_device_id *devices; + cl_context context; + cl_command_queue commandQueue; + cl_program program; + cl_mem inputBuffer; + cl_mem outputBuffer; + cl_kernel kernel; + //for opencl end + void cltldOverlapRect(int *windows, int numWindows, Rect *boundary, float *overlap); + void cltldOverlapRect_self(int *windows, int numWindows, Rect *boundary, float *overlap); + int convertToString2(const char *filename, std::string& s); - TLD(); - virtual ~TLD(); - void release(); - void selectObject(const cv::Mat &img, cv::Rect *bb); - void processImage(const cv::Mat &img); - void writeToFile(const char *path); - void readFromFile(const char *path); -}; + }; } /* namespace tld */ #endif /* TLD_H_ */ diff --git a/src/libopentld/tld/TLDUtil.cpp b/src/libopentld/tld/TLDUtil.cpp index 3587a037..b430771b 100644 --- a/src/libopentld/tld/TLDUtil.cpp +++ b/src/libopentld/tld/TLDUtil.cpp @@ -16,234 +16,312 @@ * along with OpenTLD. If not, see . * */ - +#include #include "TLDUtil.h" #include "NormalizedPatch.h" #include "DetectorCascade.h" - +#include "omp.h" using namespace std; using namespace cv; namespace tld { -void tldRectToPoints(Rect rect, CvPoint *p1, CvPoint *p2) -{ - p1->x = rect.x; - p1->y = rect.y; - p2->x = rect.x + rect.width; - p2->y = rect.y + rect.height; -} - -void tldBoundingBoxToPoints(int *bb, CvPoint *p1, CvPoint *p2) -{ - p1->x = bb[0]; - p1->y = bb[1]; - p2->x = bb[0] + bb[2]; - p2->y = bb[1] + bb[3]; -} - - - -//Returns mean-normalized patch, image must be greyscale -void tldNormalizeImg(const Mat &img, float *output) -{ - int size = TLD_PATCH_SIZE; - - Mat result; - resize(img, result, cvSize(size, size)); //Default is bilinear - - float mean = 0; - - unsigned char *imgData = (unsigned char *)result.data; - - for(int i = 0; i < 15; i++) - { - for(int j = 0; j < 15; j++) - { - mean += imgData[j * result.step + i]; - } - } - - mean /= size * size; - - - for(int i = 0; i < size; i++) - { - for(int j = 0; j < size; j++) - { - output[j * 15 + i] = imgData[j * result.step + i] - mean; - } - } - -} - -CvRect tldBoundaryToRect(int *boundary) -{ - return Rect(boundary[0], boundary[1], boundary[2], boundary[3]); -} - -void tldExtractSubImage(const Mat &img, Mat &subImage, CvRect rect) -{ - subImage = img(rect).clone(); -} + void tldRectToPoints(Rect rect, CvPoint *p1, CvPoint *p2) + { + p1->x = rect.x; + p1->y = rect.y; + p2->x = rect.x + rect.width; + p2->y = rect.y + rect.height; + } + + void tldBoundingBoxToPoints(int *bb, CvPoint *p1, CvPoint *p2) + { + p1->x = bb[0]; + p1->y = bb[1]; + p2->x = bb[0] + bb[2]; + p2->y = bb[1] + bb[3]; + } -void tldExtractSubImage(const Mat &img, Mat &subImage, int *boundary) -{ - tldExtractSubImage(img, subImage, tldBoundaryToRect(boundary)); -} -void tldExtractNormalizedPatch(const Mat &img, int x, int y, int w, int h, float *output) -{ - Mat subImage; - tldExtractSubImage(img, subImage, Rect(x, y, w, h)); - tldNormalizeImg(subImage, output); -} -//TODO: Rename -void tldExtractNormalizedPatchBB(const Mat &img, int *boundary, float *output) -{ - int x, y, w, h; - tldExtractDimsFromArray(boundary, &x, &y, &w, &h); - tldExtractNormalizedPatch(img, x, y, w, h, output); -} + //Returns mean-normalized patch, image must be greyscale + void tldNormalizeImg(const Mat &img, float *output) + { + int size = TLD_PATCH_SIZE; -void tldExtractNormalizedPatchRect(const Mat &img, Rect *rect, float *output) -{ - tldExtractNormalizedPatch(img, rect->x, rect->y, rect->width, rect->height, output); -} + Mat result; + resize(img, result, cvSize(size, size)); //Default is bilinear -float CalculateMean(float *value, int n) -{ + float mean = 0; - float sum = 0; + unsigned char *imgData = (unsigned char *)result.data; - for(int i = 0; i < n; i++) - sum += value[i]; + for (int i = 0; i < 15; i++) + { + for (int j = 0; j < 15; j++) + { + mean += imgData[j * result.step + i]; + } + } - return (sum / n); + mean /= size * size; -} -float tldCalcVariance(float *value, int n) -{ + for (int i = 0; i < size; i++) + { + for (int j = 0; j < size; j++) + { + output[j * 15 + i] = imgData[j * result.step + i] - mean; + } + } - float mean = CalculateMean(value, n); - float temp = 0; + } - for(int i = 0; i < n; i++) - { - temp += (value[i] - mean) * (value[i] - mean) ; - } + CvRect tldBoundaryToRect(int *boundary) + { + return Rect(boundary[0], boundary[1], boundary[2], boundary[3]); + } - return temp / n; + void tldExtractSubImage(const Mat &img, Mat &subImage, CvRect rect) + { + subImage = img(rect).clone(); + } -} - - -float tldBBOverlap(int *bb1, int *bb2) -{ + void tldExtractSubImage(const Mat &img, Mat &subImage, int *boundary) + { + tldExtractSubImage(img, subImage, tldBoundaryToRect(boundary)); + } - if(bb1[0] > bb2[0] + bb2[2]) - { - return 0.0; - } - - if(bb1[1] > bb2[1] + bb2[3]) - { - return 0.0; - } - - if(bb1[0] + bb1[2] < bb2[0]) - { - return 0.0; - } - - if(bb1[1] + bb1[3] < bb2[1]) - { - return 0.0; - } - - int colInt = min(bb1[0] + bb1[2], bb2[0] + bb2[2]) - max(bb1[0], bb2[0]); - int rowInt = min(bb1[1] + bb1[3], bb2[1] + bb2[3]) - max(bb1[1], bb2[1]); - - int intersection = colInt * rowInt; - int area1 = bb1[2] * bb1[3]; - int area2 = bb2[2] * bb2[3]; - return intersection / (float)(area1 + area2 - intersection); -} - -void tldOverlapOne(int *windows, int numWindows, int index, vector * indices, float *overlap) -{ - - for(size_t i = 0; i < indices->size(); i++) - { - - overlap[i] = tldBBOverlap(&windows[TLD_WINDOW_SIZE * index], &windows[TLD_WINDOW_SIZE * indices->at(i)]); - } - -} - -float tldOverlapRectRect(Rect r1, Rect r2) -{ - int bb1[4]; - int bb2[4]; - tldRectToArray(r1, bb1); - tldRectToArray(r2, bb2); - return tldBBOverlap(bb1, bb2); - -} - -Rect *tldCopyRect(Rect *r) -{ - Rect *r2 = new Rect(); - r2->x = r->x; - r2->y = r->y; - r2->width = r->width; - r2->height = r->height; - return r2; -} - -void tldOverlapRect(int *windows, int numWindows, Rect *boundary, float *overlap) -{ - int bb[4]; - bb[0] = boundary->x; - bb[1] = boundary->y; - bb[2] = boundary->width; - bb[3] = boundary->height; - - tldOverlap(windows, numWindows, bb, overlap); -} - -void tldOverlap(int *windows, int numWindows, int *boundary, float *overlap) -{ - - for(int i = 0; i < numWindows; i++) - { - - overlap[i] = tldBBOverlap(boundary, &windows[TLD_WINDOW_SIZE * i]); - } - -} - - - - -bool tldSortByOverlapDesc(pair bb1 , pair bb2) -{ - return bb1.second > bb2.second; -} - -//Checks whether bb1 is completely inside bb2 -int tldIsInside(int *bb1, int *bb2) -{ + void tldExtractNormalizedPatch(const Mat &img, int x, int y, int w, int h, float *output) + { + Mat subImage; + tldExtractSubImage(img, subImage, Rect(x, y, w, h)); + tldNormalizeImg(subImage, output); + } - if(bb1[0] > bb2[0] && bb1[1] > bb2[1] && bb1[0] + bb1[2] < bb2[0] + bb2[2] && bb1[1] + bb1[3] < bb2[1] + bb2[3]) - { - return 1; - } - else return 0; + //TODO: Rename + void tldExtractNormalizedPatchBB(const Mat &img, int *boundary, float *output) + { + int x, y, w, h; + tldExtractDimsFromArray(boundary, &x, &y, &w, &h); + tldExtractNormalizedPatch(img, x, y, w, h, output); + } + + void tldExtractNormalizedPatchRect(const Mat &img, Rect *rect, float *output) + { + tldExtractNormalizedPatch(img, rect->x, rect->y, rect->width, rect->height, output); + } + + float CalculateMean(float *value, int n) + { + + float sum = 0; + + for (int i = 0; i < n; i++) + sum += value[i]; + + return (sum / n); + + } + + float tldCalcVariance(float *value, int n) + { + + float mean = CalculateMean(value, n); + float temp = 0; + float temp2 = 0; + for (int i = 0; i < n; i++) + { + temp2 = (value[i] - mean); + temp += temp2*temp2; + } + + return temp / n; + + } + + + float tldBBOverlap(int *bb1, int *bb2) + { + + if (bb1[0] > bb2[0] + bb2[2]) + { + return 0.0; + } + + if (bb1[1] > bb2[1] + bb2[3]) + { + return 0.0; + } + + if (bb1[0] + bb1[2] < bb2[0]) + { + return 0.0; + } + + if (bb1[1] + bb1[3] < bb2[1]) + { + return 0.0; + } + + int colInt = min(bb1[0] + bb1[2], bb2[0] + bb2[2]) - max(bb1[0], bb2[0]); + int rowInt = min(bb1[1] + bb1[3], bb2[1] + bb2[3]) - max(bb1[1], bb2[1]); + + int intersection = colInt * rowInt; + int area1 = bb1[2] * bb1[3]; + int area2 = bb2[2] * bb2[3]; + return intersection / (float)(area1 + area2 - intersection); + } + + void tldOverlapOne(int *windows, int numWindows, int index, vector * indices, float *overlap) + { + + for (size_t i = 0; i < indices->size(); i++) + { + + overlap[i] = tldBBOverlap(&windows[TLD_WINDOW_SIZE * index], &windows[TLD_WINDOW_SIZE * indices->at(i)]); + } + + } + + float tldOverlapRectRect(Rect r1, Rect r2) + { + int bb1[4]; + int bb2[4]; + tldRectToArray(r1, bb1); + tldRectToArray(r2, bb2); + return tldBBOverlap(bb1, bb2); + + } + + Rect *tldCopyRect(Rect *r) + { + Rect *r2 = new Rect(); + r2->x = r->x; + r2->y = r->y; + r2->width = r->width; + r2->height = r->height; + return r2; + } + + void tldOverlapRect(int *windows, int numWindows, Rect *boundary, float *overlap) + { + int bb[4]; + bb[0] = boundary->x; + bb[1] = boundary->y; + bb[2] = boundary->width; + bb[3] = boundary->height; + + tldOverlap(windows, numWindows, bb, overlap); + } + //expanded/unfolded by xqc + void tldOverlapRect_unfold(int *windows, int numWindows, Rect *boundary, float *overlap) + { + int bb1[4], *bb2; + bb1[0] = boundary->x; + bb1[1] = boundary->y; + bb1[2] = boundary->width; + bb1[3] = boundary->height; + //tldOverlap(windows, numWindows, bb, overlap); + //void tldOverlap(int *windows, int numWindows, int *boundary, float *overlap) + //for (int i = 0; i < numWindows; i++) + //{ + // overlap[i] = tldBBOverlap(boundary, &windows[TLD_WINDOW_SIZE * i]); + //} + + + + for (int i = 0; i < numWindows; i++) + { + bb2 = windows + TLD_WINDOW_SIZE * i; + + //overlap[i] = tldBBOverlap(bb, &windows[TLD_WINDOW_SIZE * i]); + //float tldBBOverlap(int *bb1, int *bb2) + + + if (bb1[0] > bb2[0] + bb2[2]) + { + overlap[i] = 0.0; + continue; + } + + if (bb1[1] > bb2[1] + bb2[3]) + { + overlap[i] = 0.0; + continue; + } + + if (bb1[0] + bb1[2] < bb2[0]) + { + overlap[i] = 0.0; + continue; + } + + if (bb1[1] + bb1[3] < bb2[1]) + { + overlap[i] = 0.0; + continue; + } + else + { + int colInt = min(bb1[0] + bb1[2], bb2[0] + bb2[2]) - max(bb1[0], bb2[0]); + int rowInt = min(bb1[1] + bb1[3], bb2[1] + bb2[3]) - max(bb1[1], bb2[1]); + + int intersection = colInt * rowInt; + int area1 = bb1[2] * bb1[3]; + int area2 = bb2[2] * bb2[3]; + overlap[i] = intersection / (float)(area1 + area2 - intersection); + } + //if (overlap[i] != 0.0) + //{ + // printf("the index is %d, overlap[%d]=%f\n", i, i, overlap[i]); + // Sleep(1000); + //} + + //printf("windowIdx =%d ,bb[0]=%d,bb2[1]=%d,bb[2]=%d,bb[3]=%d\n", i, bb2[0], bb2[1], bb2[2], bb2[3]); Sleep(1000); + } + + } + void tldOverlap(int *windows, int numWindows, int *boundary, float *overlap) + { + + //omp_set_num_threads(4); +#pragma omp parallel for num_threads(8) + for (int i = 0; i < numWindows; i++) + { + + // int thread = omp_get_thread_num(); + // printf("thread=%d\n",thread) ; + //float temp; + //temp = tldBBOverlap(boundary, &windows[TLD_WINDOW_SIZE * i]); + //overlap[i] = temp; + overlap[i] = tldBBOverlap(boundary, &windows[TLD_WINDOW_SIZE * i]); + + + } + + + } + + + + + bool tldSortByOverlapDesc(pair bb1, pair bb2) + { + return bb1.second > bb2.second; + } + + //Checks whether bb1 is completely inside bb2 + int tldIsInside(int *bb1, int *bb2) + { + + if (bb1[0] > bb2[0] && bb1[1] > bb2[1] && bb1[0] + bb1[2] < bb2[0] + bb2[2] && bb1[1] + bb1[3] < bb2[1] + bb2[3]) + { + return 1; + } + else return 0; -} + } } /* End Namespace */ diff --git a/src/libopentld/tld/TLDUtil.h b/src/libopentld/tld/TLDUtil.h index dece7168..40cbdefd 100644 --- a/src/libopentld/tld/TLDUtil.h +++ b/src/libopentld/tld/TLDUtil.h @@ -18,11 +18,11 @@ */ /* - * util.h - * - * Created on: 30.05.2011 - * Author: Georg Nebehay - */ +* util.h +* +* Created on: 30.05.2011 +* Author: Georg Nebehay +*/ #ifndef TLDUTIL_H_ #define TLDUTIL_H_ @@ -34,86 +34,87 @@ namespace tld { -template -void tldConvertBB(T1 *src, T2 *dest) -{ - dest[0] = src[0]; - dest[1] = src[1]; - dest[2] = src[2]; - dest[3] = src[3]; -} - -template -void tldCopyBB(T *src, T *dest) -{ - tldConvertBB(src, dest); -} - -template -void tldCopyBoundaryToArray(T x, T y, T width, T height, T *array) -{ - array[0] = x; - array[1] = y; - array[2] = width; - array[3] = height; -} - -template -void tldExtractDimsFromArray(T *boundary, T *x, T *y, T *width, T *height) -{ - *x = boundary[0]; - *y = boundary[1]; - *width = boundary[2]; - *height = boundary[3]; -} - -template -void tldRectToArray(cv::Rect rect, T *boundary) -{ - boundary[0] = rect.x; - boundary[1] = rect.y; - boundary[2] = rect.width; - boundary[3] = rect.height; -} - -template -cv::Rect tldArrayToRect(T *boundary) -{ - cv::Rect rect; - rect.x = boundary[0]; - rect.y = boundary[1]; - rect.width = boundary[2]; - rect.height = boundary[3]; - - return rect; -} - - -int tldIsInside(int *bb1, int *bb2); -void tldRectToPoints(CvRect rect, CvPoint *p1, CvPoint *p2); -void tldBoundingBoxToPoints(int *bb, CvPoint *p1, CvPoint *p2); - -void tldNormalizeImg(const cv::Mat &img, float *result, int size); - -void tldExtractNormalizedPatch(const cv::Mat &img, int x, int y, int w, int h, float *output); -void tldExtractNormalizedPatchBB(const cv::Mat &img, int *boundary, float *output); -void tldExtractNormalizedPatchRect(const cv::Mat &img, cv::Rect *rect, float *output); -void tldExtractSubImage(const cv::Mat &img, cv::Mat &subImage, int *boundary); -void tldExtractSubImage(const cv::Mat &img, cv::Mat &subImage, int x, int y, int w, int h); - -float tldCalcMean(float *value, int n); -float tldCalcVariance(float *value, int n); - -bool tldSortByOverlapDesc(std::pair bb1 , std::pair bb2); -cv::Rect *tldCopyRect(cv::Rect *r); - -//TODO: Change function names -float tldOverlapRectRect(cv::Rect r1, cv::Rect r2); -void tldOverlapOne(int *windows, int numWindows, int index, std::vector * indices, float *overlap); -void tldOverlap(int *windows, int numWindows, int *boundary, float *overlap); -void tldOverlapRect(int *windows, int numWindows, cv::Rect *boundary, float *overlap); - -float tldCalcVariance(float *value, int n); + template + void tldConvertBB(T1 *src, T2 *dest) + { + dest[0] = src[0]; + dest[1] = src[1]; + dest[2] = src[2]; + dest[3] = src[3]; + } + + template + void tldCopyBB(T *src, T *dest) + { + tldConvertBB(src, dest); + } + + template + void tldCopyBoundaryToArray(T x, T y, T width, T height, T *array) + { + array[0] = x; + array[1] = y; + array[2] = width; + array[3] = height; + } + + template + void tldExtractDimsFromArray(T *boundary, T *x, T *y, T *width, T *height) + { + *x = boundary[0]; + *y = boundary[1]; + *width = boundary[2]; + *height = boundary[3]; + } + + template + void tldRectToArray(cv::Rect rect, T *boundary) + { + boundary[0] = rect.x; + boundary[1] = rect.y; + boundary[2] = rect.width; + boundary[3] = rect.height; + } + + template + cv::Rect tldArrayToRect(T *boundary) + { + cv::Rect rect; + rect.x = boundary[0]; + rect.y = boundary[1]; + rect.width = boundary[2]; + rect.height = boundary[3]; + + return rect; + } + + + int tldIsInside(int *bb1, int *bb2); + void tldRectToPoints(CvRect rect, CvPoint *p1, CvPoint *p2); + void tldBoundingBoxToPoints(int *bb, CvPoint *p1, CvPoint *p2); + + void tldNormalizeImg(const cv::Mat &img, float *result, int size); + + void tldExtractNormalizedPatch(const cv::Mat &img, int x, int y, int w, int h, float *output); + void tldExtractNormalizedPatchBB(const cv::Mat &img, int *boundary, float *output); + void tldExtractNormalizedPatchRect(const cv::Mat &img, cv::Rect *rect, float *output); + void tldExtractSubImage(const cv::Mat &img, cv::Mat &subImage, int *boundary); + void tldExtractSubImage(const cv::Mat &img, cv::Mat &subImage, int x, int y, int w, int h); + + float tldCalcMean(float *value, int n); + float tldCalcVariance(float *value, int n); + + bool tldSortByOverlapDesc(std::pair bb1, std::pair bb2); + cv::Rect *tldCopyRect(cv::Rect *r); + + //TODO: Change function names + float tldOverlapRectRect(cv::Rect r1, cv::Rect r2); + void tldOverlapOne(int *windows, int numWindows, int index, std::vector * indices, float *overlap); + void tldOverlap(int *windows, int numWindows, int *boundary, float *overlap); + void tldOverlapRect(int *windows, int numWindows, cv::Rect *boundary, float *overlap); + void tldOverlapRect_unfold(int *windows, int numWindows, cv::Rect *boundary, float *overlap); + + float tldCalcVariance(float *value, int n); #endif /* UTIL_H_ */ diff --git a/src/libopentld/tld/VarianceFilter.cpp b/src/libopentld/tld/VarianceFilter.cpp index 12858bc7..3cb0e716 100644 --- a/src/libopentld/tld/VarianceFilter.cpp +++ b/src/libopentld/tld/VarianceFilter.cpp @@ -18,84 +18,516 @@ */ /* - * VarianceFilter.cpp - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* VarianceFilter.cpp +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #include "VarianceFilter.h" #include "IntegralImage.h" #include "DetectorCascade.h" +#include "TLDUtil.h" +#include +#include +#include "switch.h" + +using namespace std; + + using namespace cv; +using namespace std; namespace tld { -VarianceFilter::VarianceFilter() -{ - enabled = true; - minVar = 0; - integralImg = NULL; - integralImg_squared = NULL; -} + VarianceFilter::VarianceFilter() + { + enabled = true; + minVar = 0; + integralImg = NULL; + integralImg_squared = NULL; -VarianceFilter::~VarianceFilter() -{ - release(); -} + } -void VarianceFilter::release() -{ - if(integralImg != NULL) delete integralImg; + VarianceFilter::~VarianceFilter() + { + release(); + } - integralImg = NULL; + void VarianceFilter::release() + { + if (integralImg != NULL) delete integralImg; - if(integralImg_squared != NULL) delete integralImg_squared; + integralImg = NULL; - integralImg_squared = NULL; -} + if (integralImg_squared != NULL) delete integralImg_squared; -float VarianceFilter::calcVariance(int *off) -{ + integralImg_squared = NULL; - int *ii1 = integralImg->data; - long long *ii2 = integralImg_squared->data; + } - float mX = (ii1[off[3]] - ii1[off[2]] - ii1[off[1]] + ii1[off[0]]) / (float) off[5]; //Sum of Area divided by area - float mX2 = (ii2[off[3]] - ii2[off[2]] - ii2[off[1]] + ii2[off[0]]) / (float) off[5]; - return mX2 - mX * mX; -} + int VarianceFilter::convertToString(const char *filename, std::string& s) + { + size_t size; + char* str; + std::fstream f(filename, (std::fstream::in | std::fstream::binary)); -void VarianceFilter::nextIteration(const Mat &img) -{ - if(!enabled) return; + if (f.is_open()) + { + size_t fileSize; + f.seekg(0, std::fstream::end); + size = fileSize = (size_t)f.tellg(); + f.seekg(0, std::fstream::beg); + str = new char[size + 1]; + if (!str) + { + f.close(); + return 0; + } - release(); + f.read(str, fileSize); + f.close(); + str[size] = '\0'; + s = str; + delete[] str; + return 0; + } + cout << "Error: failed to open file\n:" << filename << endl; + return false; + } - integralImg = new IntegralImage(img.size()); - integralImg->calcIntImg(img); + float VarianceFilter::calcVariance(int *off) + { - integralImg_squared = new IntegralImage(img.size()); - integralImg_squared->calcIntImg(img, true); -} + int *ii1 = integralImg->data; + float *ii2 = integralImg_squared->data; + + float mX = (ii1[off[3]] - ii1[off[2]] - ii1[off[1]] + ii1[off[0]]) / (float)off[5]; //Sum of Area divided by area + float mX2 = (ii2[off[3]] - ii2[off[2]] - ii2[off[1]] + ii2[off[0]]) / (float)off[5]; + return mX2 - mX * mX; + } + + void VarianceFilter::integralImag_opencv(const Mat &img) + { + + + + ///DevicesInfo devices; + //getOpenCLDevices(devices); + //setDevice(devices[SELECTED_DEVICE_ID]); + oclMat img1, Sum, sqsum; + + img1 = img; + + + cv::ocl::integral(img1, Sum, sqsum); + iSumMat = Sum; + fSqreSumMat = sqsum; + + //int *p2, *q2; + //float *fp2, *fq2; + //p2 = (int*) iSumMat.data; + //fp2 = (float*)fSqreSumMat.data; + + //for (int i = 0; i < 1; i++) + //{ + // for (int m = 0; m < 10; m++) + // { + // q2 = p2 + i*img.cols + m; + // fq2 = fp2 + i*img.cols + m; + // // printf("%f ", (float)(*q2)); + + // } + // printf("\n"); + //} + + + + //double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + //toc = toc / 1000000; + //printf(" the copy data used is %f************\n", toc);//float fps = 1 / toc; + // // const unsigned char *input = (const unsigned char *)(img.data); + + + + } + + void VarianceFilter::integralImag_extract(const Mat &src, Mat &sum, Mat &sqsum)//void VarianceFilter::integralImag_extract(const Mat &src) + { + + + int i, j; + + cv::Mat t_sum; + //cv::Mat sum; + cv::Mat t_sqsum; + //cv::Mat sqsum; + + int vlen = 4; cl_event events[2]; + int offset = 0; + int pre_invalid = 0; + int vcols = (pre_invalid + src.cols + vlen - 1) / vlen; + + int w = src.cols, h = src.rows; + int depth = src.depth() == CV_8U ? CV_32S : CV_64F; + int type = CV_MAKE_TYPE(depth, 1); + +#ifdef debug + printf("the type is %d\n", depth); +#endif + t_sum.create(src.cols, src.rows, type); + sum.create(h, w, type); + + t_sqsum.create(src.cols, src.rows, CV_32FC1); + sqsum.create(h, w, CV_32FC1); + + int sum_offset = 0; // sum.offset / vlen; + int sqsum_offset = 0; // sqsum.offset / vlen; + + + cl_mem srcdata = clCreateBuffer(context, CL_MEM_READ_ONLY, (src.cols) * (src.rows) * sizeof(uchar), NULL, NULL); + cl_mem tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(int), NULL, NULL); + cl_mem tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(float), NULL, NULL); + cl_mem sumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(int), NULL, NULL); + cl_mem sqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE, (src.cols) * (src.rows) * sizeof(float), NULL, NULL); + //use host ptr + //cl_mem srcdata = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (src.cols) * (src.rows) * sizeof(uchar), (void *)src.data, NULL); + //cl_mem tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(int), (void *)t_sum.data, NULL); + //cl_mem tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR , (src.cols) * (src.rows) * sizeof(float), (void *)t_sqsum.data, NULL); + //cl_mem sumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(int), (void *)sum.data, NULL); + //cl_mem sqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(float), (void *)sqsum.data, NULL); + + int step; + int src_step, sum_step; + int t_sum_step, sqsum_step; + + //src_step = 1920; + //t_sum_step = 4320; + //sum_step = 7680; + //sqsum_step = 7680; + int rows = src.rows; + int cols = src.cols; + src_step = src.step; + t_sum_step = t_sum.step; + sum_step = sum.step; + sqsum_step = sqsum.step; + + //printf(""); + status = clEnqueueWriteBuffer(commandQueue, srcdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(uchar), src.data, 0, NULL, NULL); + + assert(status == CL_SUCCESS); + //if (status != CL_SUCCESS) + //{ + // cout << "Error: src.data kernel_intgegral_cols EnqueueNDRangeKernel!" << endl; + //} + + status = clSetKernelArg(kernel_intgegral_cols, 0, sizeof(cl_mem), (void *)&srcdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&src.data)); + status = clSetKernelArg(kernel_intgegral_cols, 1, sizeof(cl_mem), (void *)&tsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sum.data)); + status = clSetKernelArg(kernel_intgegral_cols, 2, sizeof(cl_mem), (void *)&tsqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sqsum.data)); + status = clSetKernelArg(kernel_intgegral_cols, 3, sizeof(cl_int), (void *)&offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&offset)); + status = clSetKernelArg(kernel_intgegral_cols, 4, sizeof(cl_int), (void *)&pre_invalid);//args.push_back(make_pair(sizeof(cl_int), (void *)&pre_invalid)); + status = clSetKernelArg(kernel_intgegral_cols, 5, sizeof(cl_int), (void *)&rows);//args.push_back(make_pair(sizeof(cl_int), (void *)&src.rows)); + status = clSetKernelArg(kernel_intgegral_cols, 6, sizeof(cl_int), (void *)&cols);//args.push_back(make_pair(sizeof(cl_int), (void *)&src.cols)); + status = clSetKernelArg(kernel_intgegral_cols, 7, sizeof(cl_int), &src_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&src.step)); + status = clSetKernelArg(kernel_intgegral_cols, 8, sizeof(cl_int), &t_sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.step)); + + + + size_t gt[3] = { ((vcols + 1) / 2) * 256, 1, 1 }, lt[3] = { 256, 1, 1 }; + // //openCLExecuteKernel(src.clCxt, &imgproc_integral, "integral_cols", gt, lt, args, -1, depth); + status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_cols, 1, NULL, gt, lt, 0, NULL, &events[0]); + status = clWaitForEvents(1, &events[0]); + assert(status == CL_SUCCESS); + // + // status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_cols, 1, NULL, gt, lt, 0, NULL, NULL); + // + //if (status != CL_SUCCESS) + //{ + // cout << "Error: kernel_intgegral_cols EnqueueNDRangeKernel!" << endl; + //} + // status = clWaitForEvents(1, &events[0]); + + //if (status != CL_SUCCESS) + // { + // printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + // } + + /*status = clEnqueueReadBuffer(commandQueue, sumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueReadBuffer(commandQueue, sqsumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sqsum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueWriteBuffer(commandQueue, sumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueWriteBuffer(commandQueue, sqsumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), t_sqsum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + */ + + //clReleaseMemObject(tsumdata); + //clReleaseMemObject(tsqsumdata); + + + // tsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(int), (void *)t_sum.data, NULL); + // tsqsumdata = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (src.cols) * (src.rows) * sizeof(float), (void *)t_sqsum.data, NULL); + + + + status = clSetKernelArg(kernel_intgegral_rows, 0, sizeof(cl_mem), (void *)&tsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sum.data)); + status = clSetKernelArg(kernel_intgegral_rows, 1, sizeof(cl_mem), (void *)&tsqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&t_sqsum.data)); + status = clSetKernelArg(kernel_intgegral_rows, 2, sizeof(cl_mem), (void *)&sumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&sum.data)); + status = clSetKernelArg(kernel_intgegral_rows, 3, sizeof(cl_mem), (void *)&sqsumdata); //args.push_back(make_pair(sizeof(cl_mem), (void *)&sqsum.data)); + status = clSetKernelArg(kernel_intgegral_rows, 4, sizeof(cl_int), (void *)&t_sum.rows);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.rows)); + status = clSetKernelArg(kernel_intgegral_rows, 5, sizeof(cl_int), (void *)&t_sum.cols);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.cols)); + status = clSetKernelArg(kernel_intgegral_rows, 6, sizeof(cl_int), (void *)&t_sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&t_sum.step)); + status = clSetKernelArg(kernel_intgegral_rows, 7, sizeof(cl_int), (void *)&sum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&sum.step)); + status = clSetKernelArg(kernel_intgegral_rows, 8, sizeof(cl_int), (void *)&sqsum_step);//args.push_back(make_pair(sizeof(cl_int), (void *)&sqsum.step)); + status = clSetKernelArg(kernel_intgegral_rows, 9, sizeof(cl_int), (void *)&sum_offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&sum_offset)); + status = clSetKernelArg(kernel_intgegral_rows, 10, sizeof(cl_int), (void *)&sqsum_offset);//args.push_back(make_pair(sizeof(cl_int), (void *)&sqsum_offset)); + + size_t gt2[3] = { t_sum.cols * 32, 1, 1 }, lt2[3] = { 256, 1, 1 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel_intgegral_rows, 1, NULL, gt2, lt2, 0, NULL, &events[1]);//openCLExecuteKernel(src.clCxt, &imgproc_integral, "integral_rows", gt2, lt2, args, -1, depth); + + + //if (status != CL_SUCCESS) + //{ + // cout << "Error: kernel_intgegral_rows EnqueueNDRangeKernel!" << endl; + //} + status = clWaitForEvents(1, &events[1]); + assert(status == CL_SUCCESS); + //if (status != CL_SUCCESS) + //{ + // printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + //} + + //convert clMat to mat + status = clEnqueueReadBuffer(commandQueue, sumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), sum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + status = clEnqueueReadBuffer(commandQueue, sqsumdata, CL_FALSE, 0, (src.cols) * (src.rows) * sizeof(int), sqsum.data, 0, NULL, NULL); + assert(status == CL_SUCCESS); + + + //psum = (float*) &sqsum.data; + //for (i = 0; i (img.size()); + //integralImg_squared = new IntegralImage(img.size()); +#if PrintTime_integral + double tic = cvGetTickCount(); +#endif +#if clIntegral_m_1 + //integralImg->oclcalcIntImg(img); + //integralImg_squared->oclcalcIntImgSquare(img, true); + + oclMat temp; + temp = img; + jifentu2(temp, sum, sqsum); //(m+1,n+1)-->(m,n) + +#elif clIntegral_m + //integralImag_opencv( img); // (m,n)-->(m,n) + integralImag_extract(img, iSumMat, fSqreSumMat); + //integralImag_extract(img); + //integralImg->calcIntImg(img); + //integralImg_squared->calcIntImg(img, true); +#elif CPU + integralImg->calcIntImg(img); + integralImg_squared->calcIntImg(img, true); +#endif + + // int size = img.size().height*img.size().width; + //for (int i = 0; i < size; i++) + //{ + //#if clIntegral_m + // int img_dat = (int)iSumMat.data[i]; + // float img_dat2 = (float)fSqreSumMat.data[i]; + // int temp3 = img_dat; + //#else + // int img_dat = (int)integralImg->data[i]; + // float img_dat2 = (float)integralImg_squared->data[i]; + // int temp3 = img_dat; + // + //#endif + // } + +#if PrintTime_integral + double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + printf("*****the clIntegral time used is %f************\n", toc);//float fps = 1 / toc; +#endif + + + } + + bool VarianceFilter::filter(int i) + { + if (!enabled) return true; + + //float bboxvar = calcVariance(windowOffsets + TLD_WINDOW_OFFSET_SIZE * i); + float bboxvar; + int *off = windowOffsets + TLD_WINDOW_OFFSET_SIZE * i; + // + /// + int *ii1 = integralImg->data; + float *ii2 = integralImg_squared->data; + + float mX = (ii1[off[3]] - ii1[off[2]] - ii1[off[1]] + ii1[off[0]]) / (float)off[5]; //Sum of Area divided by area + float mX2 = (ii2[off[3]] - ii2[off[2]] - ii2[off[1]] + ii2[off[0]]) / (float)off[5]; + bboxvar = mX2 - mX * mX; + + + detectionResult->variances[i] = bboxvar; + //test for i + //if (i == 1172655) + // printf("i==%d,bboxvar=%f\n", i, bboxvar); + if (bboxvar < minVar) + { + return false; + } + + return true; + } + + + + + + + bool VarianceFilter::clfilter(const Mat &img) + { + //printf("begin using VarianceFilter clFilter*************************\n"); + + //char *kernelName = "M:\\OpenTLD\\OpenTLD-master\\OpenTLD-master\\src\\libopentld\\tld\\varianceFilter.cl"; + char *kernelName = "..\\..\\..\\src\\libopentld\\kernel\\varianceFilter_kernel.cpp"; // it is conbined to ensemble classify. + + cl_event events[1]; + string sourceStr; + status = convertToString(kernelName, sourceStr); + const char *source = sourceStr.c_str(); + size_t sourceSize[] = { strlen(source) }; + program = clCreateProgramWithSource(context, 1, &source, sourceSize, NULL); + + /*Step 6: Build program. */ + status = clBuildProgram(program, 1, &devices[SELECTED_DEVICE_ID], NULL, NULL, NULL); + //printf("mid using GPU*************************\n"); + if (status != CL_SUCCESS) + { + cout << "VarianceFilter Error: Getting platforms!" << endl; + return false; + } + + cl_mem oclbuffWindowsOffset = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (TLD_WINDOW_OFFSET_SIZE * numWindows) * sizeof(int), (void *)windowOffsets, NULL); + cl_mem oclbuffII = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (img.size().width)*(img.size().height) * sizeof(int), (void *)iSumMat.data, NULL); + cl_mem oclbuffIISqure = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (img.size().width)*(img.size().height) * sizeof(int), (void *)fSqreSumMat.data, NULL); + cl_mem oclbuffDetectionResultVarious = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)detectionResult->variances, NULL); + cl_mem oclbuffDetectionResultPosteriors = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)detectionResult->posteriors, NULL); + + cl_mem oclbuffDetectionwindowFlags = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(int), (void *)detectionResult->windowFlags, NULL); + + /*Step 8: Create kernel object */ + kernel = clCreateKernel(program, "varianceFilter", NULL); + + /*Step 9: Sets Kernel arguments.*/ + status = clSetKernelArg(kernel, 0, sizeof(cl_mem), (void *)&oclbuffWindowsOffset); + status = clSetKernelArg(kernel, 1, sizeof(cl_mem), (void *)&oclbuffII); + status = clSetKernelArg(kernel, 2, sizeof(cl_mem), (void *)&oclbuffIISqure); + status = clSetKernelArg(kernel, 3, sizeof(cl_mem), (void *)&oclbuffDetectionResultVarious); + status = clSetKernelArg(kernel, 4, sizeof(cl_mem), (void *)&oclbuffDetectionResultPosteriors); + status = clSetKernelArg(kernel, 5, sizeof(int), (void *)&numWindows); + status = clSetKernelArg(kernel, 6, sizeof(float), (void *)&minVar); + status = clSetKernelArg(kernel, 7, sizeof(cl_mem), (void*)&oclbuffDetectionwindowFlags); + //status = clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *)&TLD_WINDOW_OFFSET_SIZE); + + /*Step 10: Running the kernel.*/ + printf("begore opencl kernel numWindows=%d\n", numWindows); + size_t global_work_size[1] = { numWindows }; + size_t local_work_size[1] = { 256 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel, 1, NULL, global_work_size, NULL, 0, NULL, &events[0]); + if (status != CL_SUCCESS) + { + cout << "Error:VarianceFilter EnqueueNDRangeKernel!" << endl; + //return false; + } + status = clWaitForEvents(1, &events[0]); + + if (status != CL_SUCCESS) + { + printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + + } + //cout << "o" << endl; + + status = clReleaseEvent(events[0]); + + //printf("end using GPU*************************\n"); + //for (int i = 0; i < numWindows; i++) + // if(detectionResult->windowFlags[i]==1) + // printf("detectionResult[%d] is %d\n", i,detectionResult->windowFlags[i]); + + /*Step 11: Read the cout put back to host memory.*/ + //status = clEnqueueReadBuffer(commandQueue, outputBuffer, CL_TRUE, 0, 12 * sizeof(char), output, 0, NULL, NULL); -bool VarianceFilter::filter(int i) -{ - if(!enabled) return true; - float bboxvar = calcVariance(windowOffsets + TLD_WINDOW_OFFSET_SIZE * i); - detectionResult->variances[i] = bboxvar; + clReleaseKernel(kernel); + clReleaseProgram(program); - if(bboxvar < minVar) - { - return false; - } + clReleaseMemObject(oclbuffWindowsOffset); + clReleaseMemObject(oclbuffII); + clReleaseMemObject(oclbuffIISqure); + clReleaseMemObject(oclbuffDetectionResultVarious); + clReleaseMemObject(oclbuffDetectionResultPosteriors); + clReleaseMemObject(oclbuffDetectionwindowFlags); + return true; - return true; -} + } } /* namespace tld */ diff --git a/src/libopentld/tld/VarianceFilter.h b/src/libopentld/tld/VarianceFilter.h index 19adda3e..31739f6d 100644 --- a/src/libopentld/tld/VarianceFilter.h +++ b/src/libopentld/tld/VarianceFilter.h @@ -17,44 +17,81 @@ * */ /* - * VarianceFilter.h - * - * Created on: Nov 16, 2011 - * Author: Georg Nebehay - */ +* VarianceFilter.h +* +* Created on: Nov 16, 2011 +* Author: Georg Nebehay +*/ #ifndef VARIANCEFILTER_H_ #define VARIANCEFILTER_H_ #include +#include #include "IntegralImage.h" #include "DetectionResult.h" +#include "switch.h" namespace tld { -class VarianceFilter -{ - IntegralImage* integralImg; - IntegralImage* integralImg_squared; + class VarianceFilter + { + + + public: + oclMat clsum, clsqsum; + + + //move here from top, which was set public from private. //edited by xqc + IntegralImage* integralImg; + IntegralImage* integralImg_squared; + bool enabled; + int *windowOffsets; + + DetectionResult *detectionResult; + + float minVar; + //for opencl begin + int numWindows; + cl_uint numPlatforms; //the NO. of platforms + cl_platform_id platform; //the chosen platform + cl_int status; + cl_uint numDevices; + + + cl_device_id *devices; + cl_context context; + cl_command_queue commandQueue; + cl_program program; + cl_mem inputBuffer; + cl_mem outputBuffer; + cl_kernel kernel, kernel_intgegral_cols, kernel_intgegral_rows; + cv::Mat iSumMat; + cv::Mat fSqreSumMat; + + + + + //for opencl end + VarianceFilter(); + virtual ~VarianceFilter(); + + void release(); + void nextIteration(const cv::Mat &img); + bool filter(int idx); -public: - bool enabled; - int *windowOffsets; + bool clfilter(const Mat &img); + float calcVariance(int *off); + int VarianceFilter::convertToString(const char *filename, std::string& s); + void integralImag_opencv(const Mat &img); - DetectionResult *detectionResult; + void integralImag_extract(const Mat &src, Mat &iSumMat, Mat &fSqreSumMat); - float minVar; - VarianceFilter(); - virtual ~VarianceFilter(); - void release(); - void nextIteration(const cv::Mat &img); - bool filter(int idx); - float calcVariance(int *off); -}; + }; } /* namespace tld */ #endif /* VARIANCEFILTER_H_ */ diff --git a/src/opentld/OpenTLD.cpp b/src/opentld/OpenTLD.cpp index 554f32ed..9a936ec5 100644 --- a/src/opentld/OpenTLD.cpp +++ b/src/opentld/OpenTLD.cpp @@ -18,51 +18,107 @@ */ /** - * @author Georg Nebehay - */ +* @author Georg Nebehay +*/ #include "Main.h" #include "Config.h" #include "ImAcq.h" #include "Gui.h" + + + + + using tld::Config; using tld::Gui; using tld::Settings; +#define __NO_STD_VECTOR +#define __CL_ENABLE_EXCEPTIONS + +#include +#include +#include +using namespace std; + + + +#include +#include +#include +#include +#include + +#define SUCCESS 0 +#define FAILURE 1 + +int convertToString(const char *filename, std::string& s) +{ + size_t size; + char* str; + std::fstream f(filename, (std::fstream::in | std::fstream::binary)); + + if (f.is_open()) + { + size_t fileSize; + f.seekg(0, std::fstream::end); + size = fileSize = (size_t)f.tellg(); + f.seekg(0, std::fstream::beg); + str = new char[size + 1]; + if (!str) + { + f.close(); + return 0; + } + + f.read(str, fileSize); + f.close(); + str[size] = '\0'; + s = str; + delete[] str; + return 0; + } + cout << "Error: failed to open file\n:" << filename << endl; + return FAILURE; +} int main(int argc, char **argv) { - Main *main = new Main(); - Config config; - ImAcq *imAcq = imAcqAlloc(); - Gui *gui = new Gui(); - main->gui = gui; - main->imAcq = imAcq; - if(config.init(argc, argv) == PROGRAM_EXIT) - { - return EXIT_FAILURE; - } + Main *main = new Main(); + Config config; + ImAcq *imAcq = imAcqAlloc(); + Gui *gui = new Gui(); + + main->gui = gui; + main->imAcq = imAcq; + + if (config.init(argc, argv) == PROGRAM_EXIT) + { + return EXIT_FAILURE; + } - config.configure(main); + config.configure(main); - srand(main->seed); + srand(main->seed); - imAcqInit(imAcq); + imAcqInit(imAcq); - if(main->showOutput) - { - gui->init(); - } + if (main->showOutput) + { + gui->init(); + } main->doWork(); - main->doWork(); + delete main; + main = NULL; + delete gui; + gui = NULL; - delete main; - main = NULL; - delete gui; - gui = NULL; - return EXIT_SUCCESS; + return EXIT_SUCCESS; } + + diff --git a/src/opentld/main/Config.cpp b/src/opentld/main/Config.cpp index faec384c..0c777466 100644 --- a/src/opentld/main/Config.cpp +++ b/src/opentld/main/Config.cpp @@ -26,472 +26,472 @@ using namespace std; namespace tld { -static char help_text[] = - "usage: tld [option arguments] [arguments]\n" - "option arguments:\n" - "[-a ] video starts at the frameNumber \n" - "[-b ] Initial bounding box\n" - "[-c] shows color images instead of greyscale\n" - "[-d ] select input device: =(IMGS|CAM|VID)\n" - " IMGS: capture from images\n" - " CAM: capture from connected camera\n" - " VID: capture from a video\n" - " STREAM: capture from a stream\n" - "[-e ] export model after run to \n" - "[-f] shows foreground\n" - "[-i ] to the images or to the video\n" - "[-h] shows help\n" - "[-j ] specifies the of the last frames which are considered by the trajectory; 0 disables the trajectory\n" - "[-m ] if specified load a model from . An initialBoundingBox must be specified or selectManually must be true.\n" - "[-n ] specifies which camera device to use.\n" - "[-p ] prints results into the file \n" - "[-s] if set, user can select initial bounding box\n" - "[-t ] threshold for determining positive results\n" - "[-z ] video ends at the frameNumber .\n" - " If is 0 or the option argument isn't specified means\n" - " take all frames.\n" - "arguments:\n" - "[] to the config file\n"; - -Config::Config() : - m_selectManuallySet(false), - m_methodSet(false), - m_startFrameSet(false), - m_lastFrameSet(false), - m_trajectorySet(false), - m_showDetectionsSet(false), - m_showForegroundSet(false), - m_thetaSet(false), - m_printResultsSet(false), - m_camNoSet(false), - m_imagePathSet(false), - m_modelPathSet(false), - m_initialBBSet(false), - m_showOutputSet(false), - m_exportModelAfterRunSet(false) -{ -} - -Config::Config(Settings &settings) : - m_settings(settings) -{ -} - -Config::~Config() -{ -} - -int Config::init(int argc, char **argv) -{ - // check cli arguments - int c; - - while((c = getopt(argc, argv, "a:b:d:e:fhi:j:m:n:Op:qst:z:")) != -1) - { - switch(c) - { - case 'a': - m_settings.m_startFrame = atoi(optarg); - m_startFrameSet = true; - break; - case 'b': - char *pch; - pch = strtok(optarg, ","); - - while(pch != NULL) - { - m_settings.m_initialBoundingBox.push_back(atoi(pch)); - pch = strtok(NULL, ","); - } - - break; - case 'd': - - if(!strcmp(optarg, "CAM")) - { - m_settings.m_method = IMACQ_CAM; - m_methodSet = true; - } - else if(!strcmp(optarg, "VID")) - { - m_settings.m_method = IMACQ_VID; - m_methodSet = true; - } - else if(!strcmp(optarg, "IMGS")) - { - m_settings.m_method = IMACQ_IMGS; - m_methodSet = true; - } - else if(!strcmp(optarg, "STREAM")) - { - m_settings.m_method = IMACQ_STREAM; - m_methodSet = true; - } - - break; - case 'e': - m_settings.m_exportModelAfterRun = true; - m_settings.m_modelExportFile = optarg; - break; - case 'f': - m_settings.m_showForeground = true; - m_showForegroundSet = true; - break; - case 'h': - cout << help_text; - return PROGRAM_EXIT; - break; - case 'i': - m_settings.m_imagePath = optarg; - m_imagePathSet = true; - break; - case 'j': - m_settings.m_trajectory = atoi(optarg); - m_trajectorySet = true; - break; - case 'm': - m_settings.m_loadModel = true; - m_settings.m_modelPath = optarg; - m_modelPathSet = true; - break; - case 'n': - m_settings.m_camNo = atoi(optarg); - m_camNoSet = true; - break; - case 'p': - m_settings.m_printResults = optarg; - m_printResultsSet = true; - break; - case 'O': - m_settings.m_showOutput = false; - m_showOutputSet = true; - break; - case 's': - m_settings.m_selectManually = true; - m_selectManuallySet = true; - break; - case 't': - m_settings.m_threshold = atof(optarg); - m_thetaSet = true; - break; - case 'z': - m_settings.m_lastFrame = atoi(optarg); - m_lastFrameSet = true; - break; - } - } - - if(!m_imagePathSet && m_methodSet && (m_settings.m_method == IMACQ_VID || m_settings.m_method == IMACQ_IMGS)) - { - cerr << "Error: Must set imagePath and method if capturing from images or a video." << endl; - return PROGRAM_EXIT; - } - - if(argc > optind) - m_configPath = argv[optind]; - - - // load config file - if(!m_configPath.empty()) - { - // read config file - try - { - m_cfg.readFile(m_configPath.c_str()); - } - catch(const libconfig::FileIOException &fioex) - { - cerr << "I/O error while reading config file." << endl; - return PROGRAM_EXIT; - } - catch(const libconfig::ParseException &pex) - { - cerr << "ConfigFile: Parse error" << endl; - return PROGRAM_EXIT; - } - - // method - string method; - m_cfg.lookupValue("acq.method", method); - - // imagePath - if(method.compare("IMGS") == 0) - { - m_settings.m_method = IMACQ_IMGS; - - try - { - m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); - } - catch(const libconfig::SettingNotFoundException &nfex) - { - cerr << "Error: Unable to read image path." << endl; - return PROGRAM_EXIT; - } - } - else if(method.compare("STREAM") == 0) - { - m_settings.m_method = IMACQ_STREAM; - - try - { - m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); - } - catch(const libconfig::SettingNotFoundException &nfex) - { - cerr << "Error: Unable to read stream URL." << endl; - return PROGRAM_EXIT; - } - } - else if(method.compare("VID") == 0) - { - m_settings.m_method = IMACQ_VID; - - try - { - m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); - } - catch(const libconfig::SettingNotFoundException &nfex) - { - cerr << "Error: Unable to read image path." << endl; - return PROGRAM_EXIT; - } - } - else if(method.compare("CAM") == 0) - { - m_settings.m_method = IMACQ_CAM; - } - else if(method.compare("LIVESIM") == 0) - { - m_settings.m_method = IMACQ_LIVESIM; - //fps - m_cfg.lookupValue("acq.fps", m_settings.m_fps); - - try - { - m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); - } - catch(const libconfig::SettingNotFoundException &nfex) - { - cerr << "Error: Unable to read image path." << endl; - return PROGRAM_EXIT; - } - - - } - - // startFrame - if(!m_startFrameSet) - m_cfg.lookupValue("acq.startFrame", m_settings.m_startFrame); - - // lastFrame - if(!m_lastFrameSet) - m_cfg.lookupValue("acq.lastFrame", m_settings.m_lastFrame); - - // camNo - if(!m_camNoSet) - m_cfg.lookupValue("acq.camNo", m_settings.m_camNo); - - // loadModel - if(!m_modelPathSet) - m_cfg.lookupValue("loadModel", m_settings.m_loadModel); - - // modelPath - if(!m_modelPathSet) - m_cfg.lookupValue("modelPath", m_settings.m_modelPath); - - // check if loadModel and modelPath are set, if one of them is set - if(!m_modelPathSet) - if(m_settings.m_loadModel && m_settings.m_modelPath.empty()) - { - cerr << "Error: modelPath must be set when using loadModel" << endl; - return PROGRAM_EXIT; - } - - // useProportionalShift - m_cfg.lookupValue("detector.useProportionalShift", m_settings.m_useProportionalShift); - - // proportionalShift - m_cfg.lookupValue("detector.proportionalShift", m_settings.m_proportionalShift); - - // minScale - m_cfg.lookupValue("detector.minScale", m_settings.m_minScale); - - // maxScale - m_cfg.lookupValue("detector.maxScale", m_settings.m_maxScale); - - // minSize - m_cfg.lookupValue("detector.minSize", m_settings.m_minSize); - - // numTrees - m_cfg.lookupValue("detector.numTrees", m_settings.m_numTrees); - - // numFeatures - m_cfg.lookupValue("detector.numFeatures", m_settings.m_numFeatures); - - // numFeatures - m_cfg.lookupValue("detector.thetaP", m_settings.m_thetaP); - m_cfg.lookupValue("detector.thetaN", m_settings.m_thetaN); - - // backgroundFrame - // TODO - //const char * backgroundFrame = NULL; - //config_lookup_string(&m_config, "backgroundFrame", &backgroundFrame); - //if(backgroundFrame != NULL) { - // classifier->background = imAcqLoadImg(imAcq, (char *)backgroundFrame, CV_LOAD_IMAGE_GRAYSCALE); - //} - - // showOutput - if(!m_showOutputSet) - m_cfg.lookupValue("showOutput", m_settings.m_showOutput); - - // trajectory - if(!m_trajectorySet) - m_cfg.lookupValue("trajectory", m_settings.m_trajectory); - - // printResults - if(!m_printResultsSet) - m_cfg.lookupValue("printResults", m_settings.m_printResults); - - // printTiming - m_cfg.lookupValue("printTiming", m_settings.m_printTiming); - - // learningEnabled - m_cfg.lookupValue("learningEnabled", m_settings.m_learningEnabled); - - // trackerEnabled - m_cfg.lookupValue("trackerEnabled", m_settings.m_trackerEnabled); - - // varianceFilterEnabled - m_cfg.lookupValue("detector.varianceFilterEnabled", m_settings.m_varianceFilterEnabled); - - // emnsembleClassifierEnabled - m_cfg.lookupValue("detector.ensembleClassifierEnabled", m_settings.m_ensembleClassifierEnabled); - - // nnClassifierEnabled - m_cfg.lookupValue("detector.nnClassifierEnabled", m_settings.m_nnClassifierEnabled); - - // selectManually - if(!m_selectManuallySet) - m_cfg.lookupValue("selectManually", m_settings.m_selectManually); - - // saveDir - m_cfg.lookupValue("saveDir", m_settings.m_outputDir); - - // theta - if(!m_thetaSet) - m_cfg.lookupValue("threshold", m_settings.m_threshold); - - // showNotConfident - m_cfg.lookupValue("showNotConfident", m_settings.m_showNotConfident); - - // showForeground - if(!m_showForegroundSet) - m_cfg.lookupValue("showForeground", m_settings.m_showForeground); - - // showDetections - if(!m_showDetectionsSet) - m_cfg.lookupValue("showDetections", m_settings.m_showDetections); - - // alternating - m_cfg.lookupValue("alternating", m_settings.m_alternating); - - // exportModelFile - m_cfg.lookupValue("modelExportFile", m_settings.m_modelExportFile); - - // exportModelAfterRun - if(!m_exportModelAfterRunSet) - m_cfg.lookupValue("exportModelAfterRun", m_settings.m_exportModelAfterRun); - - // seed - m_cfg.lookupValue("seed", m_settings.m_seed); - - // initialBoundingBox - try - { - libconfig::Setting &initBB_setting = m_cfg.lookup("initialBoundingBox"); - - for(int i = 0; i < 4; i++) - { - m_settings.m_initialBoundingBox.push_back(initBB_setting[i]); - } - } - catch(const libconfig::SettingNotFoundException &nfex) - { - // Ignore - } - } - - return SUCCESS; -} - -int Config::configure(Main *main) -{ - ImAcq *imAcq = main->imAcq; - - // imAcq - imAcq->method = m_settings.m_method; - imAcq->imgPath = (m_settings.m_imagePath.empty()) ? NULL : m_settings.m_imagePath.c_str(); - imAcq->lastFrame = m_settings.m_lastFrame; - imAcq->currentFrame = m_settings.m_startFrame; - imAcq->camNo = m_settings.m_camNo; - imAcq->fps = m_settings.m_fps; - - // main - main->tld->trackerEnabled = m_settings.m_trackerEnabled; - main->showOutput = m_settings.m_showOutput; - main->showTrajectory = (m_settings.m_trajectory) ? true : false; - main->trajectoryLength = m_settings.m_trajectory; - main->printResults = (m_settings.m_printResults.empty()) ? NULL : m_settings.m_printResults.c_str(); - main->saveDir = (m_settings.m_outputDir.empty()) ? NULL : m_settings.m_outputDir.c_str(); - main->threshold = m_settings.m_threshold; - main->showForeground = m_settings.m_showForeground; - main->showNotConfident = m_settings.m_showNotConfident; - main->tld->alternating = m_settings.m_alternating; - main->tld->learningEnabled = m_settings.m_learningEnabled; - main->selectManually = m_settings.m_selectManually; - main->exportModelAfterRun = m_settings.m_exportModelAfterRun; - main->modelExportFile = m_settings.m_modelExportFile.c_str(); - main->loadModel = m_settings.m_loadModel; - main->modelPath = (m_settings.m_modelPath.empty()) ? NULL : m_settings.m_modelPath.c_str(); - main->seed = m_settings.m_seed; - - if(m_settings.m_initialBoundingBox.size() > 0) - { - main->initialBB = new int[4]; - - for(int i = 0; i < 4; i++) - { - main->initialBB[i] = m_settings.m_initialBoundingBox[i]; - } - } - - DetectorCascade *detectorCascade = main->tld->detectorCascade; - detectorCascade->varianceFilter->enabled = m_settings.m_varianceFilterEnabled; - detectorCascade->ensembleClassifier->enabled = m_settings.m_ensembleClassifierEnabled; - detectorCascade->nnClassifier->enabled = m_settings.m_nnClassifierEnabled; - - // classifier - detectorCascade->useShift = m_settings.m_useProportionalShift; - detectorCascade->shift = m_settings.m_proportionalShift; - detectorCascade->minScale = m_settings.m_minScale; - detectorCascade->maxScale = m_settings.m_maxScale; - detectorCascade->minSize = m_settings.m_minSize; - detectorCascade->numTrees = m_settings.m_numTrees; - detectorCascade->numFeatures = m_settings.m_numFeatures; - detectorCascade->nnClassifier->thetaTP = m_settings.m_thetaP; - detectorCascade->nnClassifier->thetaFP = m_settings.m_thetaN; - - return SUCCESS; -} - -/* - POSIX getopt for Windows - - AT&T Public License - - Code given out at the 1985 UNIFORUM conference in Dallas. - */ + static char help_text[] = + "usage: tld [option arguments] [arguments]\n" + "option arguments:\n" + "[-a ] video starts at the frameNumber \n" + "[-b ] Initial bounding box\n" + "[-c] shows color images instead of greyscale\n" + "[-d ] select input device: =(IMGS|CAM|VID)\n" + " IMGS: capture from images\n" + " CAM: capture from connected camera\n" + " VID: capture from a video\n" + " STREAM: capture from a stream\n" + "[-e ] export model after run to \n" + "[-f] shows foreground\n" + "[-i ] to the images or to the video\n" + "[-h] shows help\n" + "[-j ] specifies the of the last frames which are considered by the trajectory; 0 disables the trajectory\n" + "[-m ] if specified load a model from . An initialBoundingBox must be specified or selectManually must be true.\n" + "[-n ] specifies which camera device to use.\n" + "[-p ] prints results into the file \n" + "[-s] if set, user can select initial bounding box\n" + "[-t ] threshold for determining positive results\n" + "[-z ] video ends at the frameNumber .\n" + " If is 0 or the option argument isn't specified means\n" + " take all frames.\n" + "arguments:\n" + "[] to the config file\n"; + + Config::Config() : + m_selectManuallySet(false), + m_methodSet(false), + m_startFrameSet(false), + m_lastFrameSet(false), + m_trajectorySet(false), + m_showDetectionsSet(false), + m_showForegroundSet(false), + m_thetaSet(false), + m_printResultsSet(false), + m_camNoSet(false), + m_imagePathSet(false), + m_modelPathSet(false), + m_initialBBSet(false), + m_showOutputSet(false), + m_exportModelAfterRunSet(false) + { + } + + Config::Config(Settings &settings) : + m_settings(settings) + { + } + + Config::~Config() + { + } + + int Config::init(int argc, char **argv) + { + // check cli arguments + int c; + + while ((c = getopt(argc, argv, "a:b:d:e:fhi:j:m:n:Op:qst:z:")) != -1) + { + switch (c) + { + case 'a': + m_settings.m_startFrame = atoi(optarg); + m_startFrameSet = true; + break; + case 'b': + char *pch; + pch = strtok(optarg, ","); + + while (pch != NULL) + { + m_settings.m_initialBoundingBox.push_back(atoi(pch)); + pch = strtok(NULL, ","); + } + + break; + case 'd': + + if (!strcmp(optarg, "CAM")) + { + m_settings.m_method = IMACQ_CAM; + m_methodSet = true; + } + else if (!strcmp(optarg, "VID")) + { + m_settings.m_method = IMACQ_VID; + m_methodSet = true; + } + else if (!strcmp(optarg, "IMGS")) + { + m_settings.m_method = IMACQ_IMGS; + m_methodSet = true; + } + else if (!strcmp(optarg, "STREAM")) + { + m_settings.m_method = IMACQ_STREAM; + m_methodSet = true; + } + + break; + case 'e': + m_settings.m_exportModelAfterRun = true; + m_settings.m_modelExportFile = optarg; + break; + case 'f': + m_settings.m_showForeground = true; + m_showForegroundSet = true; + break; + case 'h': + cout << help_text; + return PROGRAM_EXIT; + break; + case 'i': + m_settings.m_imagePath = optarg; + m_imagePathSet = true; + break; + case 'j': + m_settings.m_trajectory = atoi(optarg); + m_trajectorySet = true; + break; + case 'm': + m_settings.m_loadModel = true; + m_settings.m_modelPath = optarg; + m_modelPathSet = true; + break; + case 'n': + m_settings.m_camNo = atoi(optarg); + m_camNoSet = true; + break; + case 'p': + m_settings.m_printResults = optarg; + m_printResultsSet = true; + break; + case 'O': + m_settings.m_showOutput = false; + m_showOutputSet = true; + break; + case 's': + m_settings.m_selectManually = true; + m_selectManuallySet = true; + break; + case 't': + m_settings.m_threshold = atof(optarg); + m_thetaSet = true; + break; + case 'z': + m_settings.m_lastFrame = atoi(optarg); + m_lastFrameSet = true; + break; + } + } + + if (!m_imagePathSet && m_methodSet && (m_settings.m_method == IMACQ_VID || m_settings.m_method == IMACQ_IMGS)) + { + cerr << "Error: Must set imagePath and method if capturing from images or a video." << endl; + return PROGRAM_EXIT; + } + + if (argc > optind) + m_configPath = argv[optind]; + + + // load config file + if (!m_configPath.empty()) + { + // read config file + try + { + m_cfg.readFile(m_configPath.c_str()); + } + catch (const libconfig::FileIOException &fioex) + { + cerr << "I/O error while reading config file." << endl; + return PROGRAM_EXIT; + } + catch (const libconfig::ParseException &pex) + { + cerr << "ConfigFile: Parse error" << endl; + return PROGRAM_EXIT; + } + + // method + string method; + m_cfg.lookupValue("acq.method", method); + + // imagePath + if (method.compare("IMGS") == 0) + { + m_settings.m_method = IMACQ_IMGS; + + try + { + m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); + } + catch (const libconfig::SettingNotFoundException &nfex) + { + cerr << "Error: Unable to read image path." << endl; + return PROGRAM_EXIT; + } + } + else if (method.compare("STREAM") == 0) + { + m_settings.m_method = IMACQ_STREAM; + + try + { + m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); + } + catch (const libconfig::SettingNotFoundException &nfex) + { + cerr << "Error: Unable to read stream URL." << endl; + return PROGRAM_EXIT; + } + } + else if (method.compare("VID") == 0) + { + m_settings.m_method = IMACQ_VID; + + try + { + m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); + } + catch (const libconfig::SettingNotFoundException &nfex) + { + cerr << "Error: Unable to read image path." << endl; + return PROGRAM_EXIT; + } + } + else if (method.compare("CAM") == 0) + { + m_settings.m_method = IMACQ_CAM; + } + else if (method.compare("LIVESIM") == 0) + { + m_settings.m_method = IMACQ_LIVESIM; + //fps + m_cfg.lookupValue("acq.fps", m_settings.m_fps); + + try + { + m_cfg.lookupValue("acq.imgPath", m_settings.m_imagePath); + } + catch (const libconfig::SettingNotFoundException &nfex) + { + cerr << "Error: Unable to read image path." << endl; + return PROGRAM_EXIT; + } + + + } + + // startFrame + if (!m_startFrameSet) + m_cfg.lookupValue("acq.startFrame", m_settings.m_startFrame); + + // lastFrame + if (!m_lastFrameSet) + m_cfg.lookupValue("acq.lastFrame", m_settings.m_lastFrame); + + // camNo + if (!m_camNoSet) + m_cfg.lookupValue("acq.camNo", m_settings.m_camNo); + + // loadModel + if (!m_modelPathSet) + m_cfg.lookupValue("loadModel", m_settings.m_loadModel); + + // modelPath + if (!m_modelPathSet) + m_cfg.lookupValue("modelPath", m_settings.m_modelPath); + + // check if loadModel and modelPath are set, if one of them is set + if (!m_modelPathSet) + if (m_settings.m_loadModel && m_settings.m_modelPath.empty()) + { + cerr << "Error: modelPath must be set when using loadModel" << endl; + return PROGRAM_EXIT; + } + + // useProportionalShift + m_cfg.lookupValue("detector.useProportionalShift", m_settings.m_useProportionalShift); + + // proportionalShift + m_cfg.lookupValue("detector.proportionalShift", m_settings.m_proportionalShift); + + // minScale + m_cfg.lookupValue("detector.minScale", m_settings.m_minScale); + + // maxScale + m_cfg.lookupValue("detector.maxScale", m_settings.m_maxScale); + + // minSize + m_cfg.lookupValue("detector.minSize", m_settings.m_minSize); + + // numTrees + m_cfg.lookupValue("detector.numTrees", m_settings.m_numTrees); + + // numFeatures + m_cfg.lookupValue("detector.numFeatures", m_settings.m_numFeatures); + + // numFeatures + m_cfg.lookupValue("detector.thetaP", m_settings.m_thetaP); + m_cfg.lookupValue("detector.thetaN", m_settings.m_thetaN); + + // backgroundFrame + // TODO + //const char * backgroundFrame = NULL; + //config_lookup_string(&m_config, "backgroundFrame", &backgroundFrame); + //if(backgroundFrame != NULL) { + // classifier->background = imAcqLoadImg(imAcq, (char *)backgroundFrame, CV_LOAD_IMAGE_GRAYSCALE); + //} + + // showOutput + if (!m_showOutputSet) + m_cfg.lookupValue("showOutput", m_settings.m_showOutput); + + // trajectory + if (!m_trajectorySet) + m_cfg.lookupValue("trajectory", m_settings.m_trajectory); + + // printResults + if (!m_printResultsSet) + m_cfg.lookupValue("printResults", m_settings.m_printResults); + + // printTiming + m_cfg.lookupValue("printTiming", m_settings.m_printTiming); + + // learningEnabled + m_cfg.lookupValue("learningEnabled", m_settings.m_learningEnabled); + + // trackerEnabled + m_cfg.lookupValue("trackerEnabled", m_settings.m_trackerEnabled); + + // varianceFilterEnabled + m_cfg.lookupValue("detector.varianceFilterEnabled", m_settings.m_varianceFilterEnabled); + + // emnsembleClassifierEnabled + m_cfg.lookupValue("detector.ensembleClassifierEnabled", m_settings.m_ensembleClassifierEnabled); + + // nnClassifierEnabled + m_cfg.lookupValue("detector.nnClassifierEnabled", m_settings.m_nnClassifierEnabled); + + // selectManually + if (!m_selectManuallySet) + m_cfg.lookupValue("selectManually", m_settings.m_selectManually); + + // saveDir + m_cfg.lookupValue("saveDir", m_settings.m_outputDir); + + // theta + if (!m_thetaSet) + m_cfg.lookupValue("threshold", m_settings.m_threshold); + + // showNotConfident + m_cfg.lookupValue("showNotConfident", m_settings.m_showNotConfident); + + // showForeground + if (!m_showForegroundSet) + m_cfg.lookupValue("showForeground", m_settings.m_showForeground); + + // showDetections + if (!m_showDetectionsSet) + m_cfg.lookupValue("showDetections", m_settings.m_showDetections); + + // alternating + m_cfg.lookupValue("alternating", m_settings.m_alternating); + + // exportModelFile + m_cfg.lookupValue("modelExportFile", m_settings.m_modelExportFile); + + // exportModelAfterRun + if (!m_exportModelAfterRunSet) + m_cfg.lookupValue("exportModelAfterRun", m_settings.m_exportModelAfterRun); + + // seed + m_cfg.lookupValue("seed", m_settings.m_seed); + + // initialBoundingBox + try + { + libconfig::Setting &initBB_setting = m_cfg.lookup("initialBoundingBox"); + + for (int i = 0; i < 4; i++) + { + m_settings.m_initialBoundingBox.push_back(initBB_setting[i]); + } + } + catch (const libconfig::SettingNotFoundException &nfex) + { + // Ignore + } + } + + return SUCCESS; + } + + int Config::configure(Main *main) + { + ImAcq *imAcq = main->imAcq; + + // imAcq + imAcq->method = m_settings.m_method; + imAcq->imgPath = (m_settings.m_imagePath.empty()) ? NULL : m_settings.m_imagePath.c_str(); + imAcq->lastFrame = m_settings.m_lastFrame; + imAcq->currentFrame = m_settings.m_startFrame; + imAcq->camNo = m_settings.m_camNo; + imAcq->fps = m_settings.m_fps; + + // main + main->tld->trackerEnabled = m_settings.m_trackerEnabled; + main->showOutput = m_settings.m_showOutput; + main->showTrajectory = (m_settings.m_trajectory) ? true : false; + main->trajectoryLength = m_settings.m_trajectory; + main->printResults = (m_settings.m_printResults.empty()) ? NULL : m_settings.m_printResults.c_str(); + main->saveDir = (m_settings.m_outputDir.empty()) ? NULL : m_settings.m_outputDir.c_str(); + main->threshold = m_settings.m_threshold; + main->showForeground = m_settings.m_showForeground; + main->showNotConfident = m_settings.m_showNotConfident; + main->tld->alternating = m_settings.m_alternating; + main->tld->learningEnabled = m_settings.m_learningEnabled; + main->selectManually = m_settings.m_selectManually; + main->exportModelAfterRun = m_settings.m_exportModelAfterRun; + main->modelExportFile = m_settings.m_modelExportFile.c_str(); + main->loadModel = m_settings.m_loadModel; + main->modelPath = (m_settings.m_modelPath.empty()) ? NULL : m_settings.m_modelPath.c_str(); + main->seed = m_settings.m_seed; + + if (m_settings.m_initialBoundingBox.size() > 0) + { + main->initialBB = new int[4]; + + for (int i = 0; i < 4; i++) + { + main->initialBB[i] = m_settings.m_initialBoundingBox[i]; + } + } + + DetectorCascade *detectorCascade = main->tld->detectorCascade; + detectorCascade->varianceFilter->enabled = m_settings.m_varianceFilterEnabled; + detectorCascade->ensembleClassifier->enabled = m_settings.m_ensembleClassifierEnabled; + detectorCascade->nnClassifier->enabled = m_settings.m_nnClassifierEnabled; + + // classifier + detectorCascade->useShift = m_settings.m_useProportionalShift; + detectorCascade->shift = m_settings.m_proportionalShift; + detectorCascade->minScale = m_settings.m_minScale; + detectorCascade->maxScale = m_settings.m_maxScale; + detectorCascade->minSize = m_settings.m_minSize; + detectorCascade->numTrees = m_settings.m_numTrees; + detectorCascade->numFeatures = m_settings.m_numFeatures; + detectorCascade->nnClassifier->thetaTP = m_settings.m_thetaP; + detectorCascade->nnClassifier->thetaFP = m_settings.m_thetaN; + + return SUCCESS; + } + + /* + POSIX getopt for Windows + + AT&T Public License + + Code given out at the 1985 UNIFORUM conference in Dallas. + */ #ifndef __GNUC__ #define NULL 0 @@ -501,69 +501,69 @@ int Config::configure(Main *main) fputs(s, stderr);\ fputc(c, stderr);} -int opterr = 1; -int optind = 1; -int optopt; -char *optarg; - -int getopt(int argc, char **argv, char *opts) -{ - static int sp = 1; - register int c; - register char *cp; - - if(sp == 1) - if(optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0') - return (EOF); - else if(strcmp(argv[optind], "--") == NULL) - { - optind++; - return (EOF); - } - - optopt = c = argv[optind][sp]; - - if(c == ':' || (cp = strchr(opts, c)) == NULL) - { - ERR(": illegal option -- ", c); - - if(argv[optind][++sp] == '\0') - { - optind++; - sp = 1; - } - - return ('?'); - } - - if(*++cp == ':') - { - if(argv[optind][sp + 1] != '\0') - optarg = &argv[optind++][sp + 1]; - else if(++optind >= argc) - { - ERR(": option requires an argument -- ", c); - sp = 1; - return ('?'); - } - else - optarg = argv[optind++]; - - sp = 1; - } - else - { - if(argv[optind][++sp] == '\0') - { - sp = 1; - optind++; - } - - optarg = NULL; - } - - return (c); -} + int opterr = 1; + int optind = 1; + int optopt; + char *optarg; + + int getopt(int argc, char **argv, char *opts) + { + static int sp = 1; + register int c; + register char *cp; + + if (sp == 1) + if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0') + return (EOF); + else if (strcmp(argv[optind], "--") == NULL) + { + optind++; + return (EOF); + } + + optopt = c = argv[optind][sp]; + + if (c == ':' || (cp = strchr(opts, c)) == NULL) + { + ERR(": illegal option -- ", c); + + if (argv[optind][++sp] == '\0') + { + optind++; + sp = 1; + } + + return ('?'); + } + + if (*++cp == ':') + { + if (argv[optind][sp + 1] != '\0') + optarg = &argv[optind++][sp + 1]; + else if (++optind >= argc) + { + ERR(": option requires an argument -- ", c); + sp = 1; + return ('?'); + } + else + optarg = argv[optind++]; + + sp = 1; + } + else + { + if (argv[optind][++sp] == '\0') + { + sp = 1; + optind++; + } + + optarg = NULL; + } + + return (c); + } #endif /* __GNUC__ */ diff --git a/src/opentld/main/Gui.cpp b/src/opentld/main/Gui.cpp index 2f377286..a247c02c 100644 --- a/src/opentld/main/Gui.cpp +++ b/src/opentld/main/Gui.cpp @@ -18,15 +18,16 @@ */ /* - * gui.cpp - * - * Created on: Oct 18, 2011 - * Author: clemensk - */ +* gui.cpp +* +* Created on: Oct 18, 2011 +* Author: clemensk +*/ #include "Gui.h" #include "Main.h" +#include #include @@ -35,120 +36,151 @@ using std::string; namespace tld { -Gui::Gui() : - m_window_name("tld") -{ -} + Gui::Gui() : + m_window_name("tld") + { + } -Gui::~Gui() -{ -} + Gui::~Gui() + { + } -void Gui::init() -{ - cvNamedWindow(m_window_name.c_str(), CV_WINDOW_AUTOSIZE); - cvMoveWindow(m_window_name.c_str(), 100, 100); -} + void Gui::init() + { + cvNamedWindow(m_window_name.c_str(), CV_WINDOW_AUTOSIZE); + cvMoveWindow(m_window_name.c_str(), 100, 100); + } -void Gui::showImage(IplImage *image) -{ - cvShowImage(m_window_name.c_str(), image); -} + void Gui::showImage(IplImage *image) + { + cvShowImage(m_window_name.c_str(), image); + } + + char Gui::getKey() + { + return cvWaitKey(10); + } -char Gui::getKey() -{ - return cvWaitKey(10); -} + std::string Gui::windowName() + { + return m_window_name; + } -std::string Gui::windowName() -{ - return m_window_name; -} + static string window_name; + static CvFont font; + static IplImage *img0; + static IplImage *img1; + static CvPoint point; + static CvRect *bb; + static int drag = 0; + + static void mouseHandler(int event, int x, int y, int flags, void *param) + { + /* user press left button */ + if (event == CV_EVENT_LBUTTONDOWN && !drag) + { + point = cvPoint(x, y); + drag = 1; + } + + /* user drag the mouse */ + if (event == CV_EVENT_MOUSEMOVE && drag) + { + img1 = (IplImage *)cvClone(img0); + + cvRectangle(img1, point, cvPoint(x, y), CV_RGB(255, 0, 0), 1, 8, 0); + + cvShowImage(window_name.c_str(), img1); + cvReleaseImage(&img1); + } + + /* user release left button */ + if (event == CV_EVENT_LBUTTONUP && drag) + { + *bb = cvRect(point.x, point.y, x - point.x, y - point.y); + drag = 0; + } + } + + // TODO: member of Gui + // --> problem: callback function mouseHandler as member! + int getBBFromUser(IplImage *img, CvRect &rect, Gui *gui) + { + //IplImage* dst; + window_name = gui->windowName(); + img0 = (IplImage *)cvClone(img); + rect = cvRect(-1, -1, -1, -1); + bb = ▭ + bool correctBB = false; + cvInitFont(&font, CV_FONT_HERSHEY_SIMPLEX, 0.5, 0.5, 0, 1, 8); + + cvSetMouseCallback(window_name.c_str(), mouseHandler, NULL); + cvPutText(img0, "Draw a bounding box and press Enter", cvPoint(0, 60), + &font, cvScalar(255, 255, 0)); + + + cvShowImage(window_name.c_str(), img0); + + + + //end dfafdads + + + + + + + while (!correctBB) + { + char key = cvWaitKey(0); + + if (tolower(key) == 'q') + { + return PROGRAM_EXIT; + } + + if (((key == '\n') || (key == '\r') || (key == '\r\n')) && (bb->x != -1) && (bb->y != -1)) + { + correctBB = true; + } + } + + if (rect.width < 0) + { + rect.x += rect.width; + rect.width = abs(rect.width); + } + + if (rect.height < 0) + { + rect.y += rect.height; + rect.height = abs(rect.height); + } + + cvSetMouseCallback(window_name.c_str(), NULL, NULL); + //cvShowImage(window_name.c_str(), img0); + + //dfafdads + //cvNamedWindow("ԴͼÏñ", 1); + //cvShowImage("ԴͼÏñ", img0); + //cvSetImageROI(img0, cvRect(100, 100, 0.5*img0->width, 0.5*img0->height)); + cvSetImageROI(img0, cvRect(rect.x, rect.y, rect.width, rect.height)); -static string window_name; -static CvFont font; -static IplImage *img0; -static IplImage *img1; -static CvPoint point; -static CvRect *bb; -static int drag = 0; + //dst = cvCreateImage(cvSize(rect.width,rect.height), IPL_DEPTH_8U, img0->nChannels); + //cvCopy(img0, dst, 0); + cvResetImageROI(img0); -static void mouseHandler(int event, int x, int y, int flags, void *param) -{ - /* user press left button */ - if(event == CV_EVENT_LBUTTONDOWN && !drag) - { - point = cvPoint(x, y); - drag = 1; - } - - /* user drag the mouse */ - if(event == CV_EVENT_MOUSEMOVE && drag) - { - img1 = (IplImage *) cvClone(img0); - - cvRectangle(img1, point, cvPoint(x, y), CV_RGB(255, 0, 0), 1, 8, 0); - - cvShowImage(window_name.c_str(), img1); - cvReleaseImage(&img1); - } - - /* user release left button */ - if(event == CV_EVENT_LBUTTONUP && drag) - { - *bb = cvRect(point.x, point.y, x - point.x, y - point.y); - drag = 0; - } -} + //cvNamedWindow("²Ù×÷ºóµÄͼÏñ", 1); + //cvShowImage("²Ù×÷ºóµÄͼÏñ", dst); + //cvDestroyWindow("²Ù×÷ºóµÄͼÏñ"); + //cvDestroyWindow("ԴͼÏñ"); + //cvSaveImage("crop.jpg", dst); + //Sleep(1000); -// TODO: member of Gui -// --> problem: callback function mouseHandler as member! -int getBBFromUser(IplImage *img, CvRect &rect, Gui *gui) -{ - window_name = gui->windowName(); - img0 = (IplImage *) cvClone(img); - rect = cvRect(-1, -1, -1, -1); - bb = ▭ - bool correctBB = false; - cvInitFont(&font, CV_FONT_HERSHEY_SIMPLEX, 0.5, 0.5, 0, 1, 8); - - cvSetMouseCallback(window_name.c_str(), mouseHandler, NULL); - cvPutText(img0, "Draw a bounding box and press Enter", cvPoint(0, 60), - &font, cvScalar(255, 255, 0)); - cvShowImage(window_name.c_str(), img0); - - while(!correctBB) - { - char key = cvWaitKey(0); - - if(tolower(key) == 'q') - { - return PROGRAM_EXIT; - } - - if(((key == '\n') || (key == '\r') || (key == '\r\n')) && (bb->x != -1) && (bb->y != -1)) - { - correctBB = true; - } - } - - if(rect.width < 0) - { - rect.x += rect.width; - rect.width = abs(rect.width); - } - - if(rect.height < 0) - { - rect.y += rect.height; - rect.height = abs(rect.height); - } - - cvSetMouseCallback(window_name.c_str(), NULL, NULL); - - cvReleaseImage(&img0); - cvReleaseImage(&img1); - - return SUCCESS; -} + //cvReleaseImage(&dst); + cvReleaseImage(&img0); + cvReleaseImage(&img1); + + return SUCCESS; + } } diff --git a/src/opentld/main/Main.cpp b/src/opentld/main/Main.cpp index d0584797..b57ad4c6 100644 --- a/src/opentld/main/Main.cpp +++ b/src/opentld/main/Main.cpp @@ -17,11 +17,11 @@ * */ /* - * MainX.cpp - * - * Created on: Nov 17, 2011 - * Author: Georg Nebehay - */ +* MainX.cpp +* +* Created on: Nov 17, 2011 +* Author: Georg Nebehay +*/ #include "Main.h" @@ -36,266 +36,279 @@ using namespace cv; void Main::doWork() { + IplImage* dst; Trajectory trajectory; - IplImage *img = imAcqGetImg(imAcq); - Mat grey(img->height, img->width, CV_8UC1); - cvtColor(cvarrToMat(img), grey, CV_BGR2GRAY); + IplImage *img = imAcqGetImg(imAcq); + Mat grey(img->height, img->width, CV_8UC1); + cvtColor(cvarrToMat(img), grey, CV_BGR2GRAY); - tld->detectorCascade->imgWidth = grey.cols; - tld->detectorCascade->imgHeight = grey.rows; - tld->detectorCascade->imgWidthStep = grey.step; + tld->detectorCascade->imgWidth = grey.cols; + tld->detectorCascade->imgHeight = grey.rows; + tld->detectorCascade->imgWidthStep = grey.step; - if(showTrajectory) + if (showTrajectory) { trajectory.init(trajectoryLength); } - if(selectManually) - { - - CvRect box; - - if(getBBFromUser(img, box, gui) == PROGRAM_EXIT) - { - return; - } - - if(initialBB == NULL) - { - initialBB = new int[4]; - } - - initialBB[0] = box.x; - initialBB[1] = box.y; - initialBB[2] = box.width; - initialBB[3] = box.height; - } - - FILE *resultsFile = NULL; - - if(printResults != NULL) - { - resultsFile = fopen(printResults, "w"); - if(!resultsFile) - { - fprintf(stderr, "Error: Unable to create results-file \"%s\"\n", printResults); - exit(-1); - } - } - - bool reuseFrameOnce = false; - bool skipProcessingOnce = false; - - if(loadModel && modelPath != NULL) - { - tld->readFromFile(modelPath); - reuseFrameOnce = true; - } - else if(initialBB != NULL) - { - Rect bb = tldArrayToRect(initialBB); - - printf("Starting at %d %d %d %d\n", bb.x, bb.y, bb.width, bb.height); - - tld->selectObject(grey, &bb); - skipProcessingOnce = true; - reuseFrameOnce = true; - } - - while(imAcqHasMoreFrames(imAcq)) - { - double tic = cvGetTickCount(); - - if(!reuseFrameOnce) - { - cvReleaseImage(&img); - img = imAcqGetImg(imAcq); - - if(img == NULL) - { - printf("current image is NULL, assuming end of input.\n"); - break; - } - - cvtColor(cvarrToMat(img), grey, CV_BGR2GRAY); - } - - if(!skipProcessingOnce) - { - tld->processImage(cvarrToMat(img)); - } - else - { - skipProcessingOnce = false; - } - - if(printResults != NULL) - { - if(tld->currBB != NULL) - { - fprintf(resultsFile, "%d %.2d %.2d %.2d %.2d %f\n", imAcq->currentFrame - 1, tld->currBB->x, tld->currBB->y, tld->currBB->width, tld->currBB->height, tld->currConf); - } - else - { - fprintf(resultsFile, "%d NaN NaN NaN NaN NaN\n", imAcq->currentFrame - 1); - } - } - - double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); - - toc = toc / 1000000; - - float fps = 1 / toc; - - int confident = (tld->currConf >= threshold) ? 1 : 0; - - if(showOutput || saveDir != NULL) - { - char string[128]; - - char learningString[10] = ""; - - if(tld->learning) - { - strcpy(learningString, "Learning"); - } - - sprintf(string, "#%d,Posterior %.2f; fps: %.2f, #numwindows:%d, %s", imAcq->currentFrame - 1, tld->currConf, fps, tld->detectorCascade->numWindows, learningString); - CvScalar yellow = CV_RGB(255, 255, 0); - CvScalar blue = CV_RGB(0, 0, 255); - CvScalar black = CV_RGB(0, 0, 0); - CvScalar white = CV_RGB(255, 255, 255); - - if(tld->currBB != NULL) - { - CvScalar rectangleColor = (confident) ? blue : yellow; - cvRectangle(img, tld->currBB->tl(), tld->currBB->br(), rectangleColor, 8, 8, 0); - - if(showTrajectory) + if (selectManually = 1) + { + + CvRect box; + + if (getBBFromUser(img, box, gui) == PROGRAM_EXIT) + { + return; + } + + if (initialBB == NULL) + { + initialBB = new int[4]; + } + + initialBB[0] = box.x; + initialBB[1] = box.y; + initialBB[2] = box.width; + initialBB[3] = box.height; + //initialBB[0] = 650; + //initialBB[1] = 550; + //initialBB[2] = 300; + //initialBB[3] = 300; + //initialBB[0] = 120; + //initialBB[1] = 120; + //initialBB[2] = 110; + //initialBB[3] = 90; + + + + } + + FILE *resultsFile = NULL; + + if (printResults != NULL) + { + resultsFile = fopen(printResults, "w"); + if (!resultsFile) + { + fprintf(stderr, "Error: Unable to create results-file \"%s\"\n", printResults); + exit(-1); + } + } + + bool reuseFrameOnce = false; + bool skipProcessingOnce = false; + + if (loadModel && modelPath != NULL) + { + tld->readFromFile(modelPath); + reuseFrameOnce = true; + } + else if (initialBB != NULL) + { + Rect bb = tldArrayToRect(initialBB); + + printf("****************Starting at %d %d %d %d***********\n", bb.x, bb.y, bb.width, bb.height); + + tld->selectObject(grey, &bb); + skipProcessingOnce = true; + reuseFrameOnce = true; + } + + while (imAcqHasMoreFrames(imAcq)) + { + double tic = cvGetTickCount(); + + if (!reuseFrameOnce) + { + cvReleaseImage(&img); + img = imAcqGetImg(imAcq); + + if (img == NULL) + { + printf("current image is NULL, assuming end of input.\n"); + break; + } + + cvtColor(cvarrToMat(img), grey, CV_BGR2GRAY); + } + + if (!skipProcessingOnce) + { + tld->detectorCascade->FramNum = imAcq->currentFrame - 1; + tld->processImage(cvarrToMat(img)); + } + else + { + skipProcessingOnce = false; + } + + if (printResults != NULL) + { + if (tld->currBB != NULL) + { + fprintf(resultsFile, "%d %.2d %.2d %.2d %.2d %f\n", imAcq->currentFrame - 1, tld->currBB->x, tld->currBB->y, tld->currBB->width, tld->currBB->height, tld->currConf); + } + else + { + fprintf(resultsFile, "%d NaN NaN NaN NaN NaN\n", imAcq->currentFrame - 1); + } + } + + double toc = (cvGetTickCount() - tic) / cvGetTickFrequency(); + + toc = toc / 1000000; + + float fps = 1 / toc; + + int confident = (tld->currConf >= threshold) ? 1 : 0; + + if (showOutput || saveDir != NULL) + { + char string[128]; + + char learningString[10] = ""; + + if (tld->learning) + { + strcpy(learningString, "Learning"); + } + + sprintf(string, "#%d,Posterior %.2f; fps: %.2f, #numwindows:%d, %s", imAcq->currentFrame - 1, tld->currConf, fps, tld->detectorCascade->numWindows, learningString); + CvScalar yellow = CV_RGB(255, 255, 0); + CvScalar blue = CV_RGB(0, 0, 255); + CvScalar black = CV_RGB(0, 0, 0); + CvScalar white = CV_RGB(255, 255, 255); + + if (tld->currBB != NULL) + { + CvScalar rectangleColor = (confident) ? blue : yellow; + cvRectangle(img, tld->currBB->tl(), tld->currBB->br(), rectangleColor, 8, 8, 0); + + if (showTrajectory) { - CvPoint center = cvPoint(tld->currBB->x+tld->currBB->width/2, tld->currBB->y+tld->currBB->height/2); - cvLine(img, cvPoint(center.x-2, center.y-2), cvPoint(center.x+2, center.y+2), rectangleColor, 2); - cvLine(img, cvPoint(center.x-2, center.y+2), cvPoint(center.x+2, center.y-2), rectangleColor, 2); + CvPoint center = cvPoint(tld->currBB->x + tld->currBB->width / 2, tld->currBB->y + tld->currBB->height / 2); + cvLine(img, cvPoint(center.x - 2, center.y - 2), cvPoint(center.x + 2, center.y + 2), rectangleColor, 2); + cvLine(img, cvPoint(center.x - 2, center.y + 2), cvPoint(center.x + 2, center.y - 2), rectangleColor, 2); trajectory.addPoint(center, rectangleColor); } - } - else if(showTrajectory) + } + else if (showTrajectory) { trajectory.addPoint(cvPoint(-1, -1), cvScalar(-1, -1, -1)); } - if(showTrajectory) + if (showTrajectory) { trajectory.drawTrajectory(img); } - CvFont font; - cvInitFont(&font, CV_FONT_HERSHEY_SIMPLEX, .5, .5, 0, 1, 8); - cvRectangle(img, cvPoint(0, 0), cvPoint(img->width, 50), black, CV_FILLED, 8, 0); - cvPutText(img, string, cvPoint(25, 25), &font, white); - - if(showForeground) - { - - for(size_t i = 0; i < tld->detectorCascade->detectionResult->fgList->size(); i++) - { - Rect r = tld->detectorCascade->detectionResult->fgList->at(i); - cvRectangle(img, r.tl(), r.br(), white, 1); - } - - } - - - if(showOutput) - { - gui->showImage(img); - char key = gui->getKey(); - - if(key == 'q') break; - - if(key == 'b') - { - - ForegroundDetector *fg = tld->detectorCascade->foregroundDetector; - - if(fg->bgImg.empty()) - { - fg->bgImg = grey.clone(); - } - else - { - fg->bgImg.release(); - } - } - - if(key == 'c') - { - //clear everything - tld->release(); - } - - if(key == 'l') - { - tld->learningEnabled = !tld->learningEnabled; - printf("LearningEnabled: %d\n", tld->learningEnabled); - } - - if(key == 'a') - { - tld->alternating = !tld->alternating; - printf("alternating: %d\n", tld->alternating); - } - - if(key == 'e') - { - tld->writeToFile(modelExportFile); - } - - if(key == 'i') - { - tld->readFromFile(modelPath); - } - - if(key == 'r') - { - CvRect box; - - if(getBBFromUser(img, box, gui) == PROGRAM_EXIT) - { - break; - } - - Rect r = Rect(box); - - tld->selectObject(grey, &r); - } - } - - if(saveDir != NULL) - { - char fileName[256]; - sprintf(fileName, "%s/%.5d.png", saveDir, imAcq->currentFrame - 1); - - cvSaveImage(fileName, img); - } - } - - if(reuseFrameOnce) - { - reuseFrameOnce = false; - } - } - - cvReleaseImage(&img); - img = NULL; - - if(exportModelAfterRun) - { - tld->writeToFile(modelExportFile); - } - - if(resultsFile) - { - fclose(resultsFile); - } + CvFont font; + cvInitFont(&font, CV_FONT_HERSHEY_SIMPLEX, .5, .5, 0, 1, 8); + cvRectangle(img, cvPoint(0, 0), cvPoint(img->width, 50), black, CV_FILLED, 8, 0); + cvPutText(img, string, cvPoint(25, 25), &font, white); + + if (showForeground) + { + + for (size_t i = 0; i < tld->detectorCascade->detectionResult->fgList->size(); i++) + { + Rect r = tld->detectorCascade->detectionResult->fgList->at(i); + cvRectangle(img, r.tl(), r.br(), white, 1); + } + + } + + + if (showOutput) + { + gui->showImage(img); + char key = gui->getKey(); + + if (key == 'q') break; + + if (key == 'b') + { + + ForegroundDetector *fg = tld->detectorCascade->foregroundDetector; + + if (fg->bgImg.empty()) + { + fg->bgImg = grey.clone(); + } + else + { + fg->bgImg.release(); + } + } + + if (key == 'c') + { + //clear everything + tld->release(); + } + + if (key == 'l') + { + tld->learningEnabled = !tld->learningEnabled; + printf("LearningEnabled: %d\n", tld->learningEnabled); + } + + if (key == 'a') + { + tld->alternating = !tld->alternating; + printf("alternating: %d\n", tld->alternating); + } + + if (key == 'e') + { + tld->writeToFile(modelExportFile); + } + + if (key == 'i') + { + tld->readFromFile(modelPath); + } + + if (key == 'r') + { + CvRect box; + + if (getBBFromUser(img, box, gui) == PROGRAM_EXIT) + { + break; + } + + Rect r = Rect(box); + + tld->selectObject(grey, &r); + } + } + + if (saveDir != NULL) + { + char fileName[256]; + sprintf(fileName, "%s/%.5d.png", saveDir, imAcq->currentFrame - 1); + + cvSaveImage(fileName, img); + } + } + + if (reuseFrameOnce) + { + reuseFrameOnce = false; + } + } + + cvReleaseImage(&img); + img = NULL; + + if (exportModelAfterRun) + { + tld->writeToFile(modelExportFile); + } + + if (resultsFile) + { + fclose(resultsFile); + } } diff --git a/src/opentld/main/Main.h b/src/opentld/main/Main.h index 3c73ee9c..d660b0f2 100644 --- a/src/opentld/main/Main.h +++ b/src/opentld/main/Main.h @@ -72,7 +72,7 @@ class Main showTrajectory = false; trajectoryLength = 0; - selectManually = 0; + selectManually = 1; initialBB = NULL; showNotConfident = true; From 814f17bfb3d9047ef76df516e5df95cef8a86781 Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Mon, 11 Sep 2017 16:58:46 +0800 Subject: [PATCH 07/14] Modified CMakeList files modified cmakelist files to find opencl include files and link OpenCL library. --- CMakeLists.txt | 3 ++- OpenTLDConfig.cmake.in | 3 ++- src/libopentld/CMakeLists.txt | 8 +++++--- src/opentld/CMakeLists.txt | 12 +++++++----- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cf377ddd..e31993a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,11 +22,12 @@ # 5) build the project "PACKAGE" to create an NSIS-installer (NSIS is required) -project(OpenTLD) +project(OpenTLD-OCL) cmake_minimum_required(VERSION 2.6) find_package(OpenCV REQUIRED) +find_package(OpenCL REQUIRED) if(BUILD_QOPENTLD) find_package(Qt4 REQUIRED) diff --git a/OpenTLDConfig.cmake.in b/OpenTLDConfig.cmake.in index 0d941c2e..5fcec247 100644 --- a/OpenTLDConfig.cmake.in +++ b/OpenTLDConfig.cmake.in @@ -1,5 +1,6 @@ -find_package(OpenCV) +find_package(OpenCV) +find_package(OpenCL) set(bin_dir "@PROJECT_BINARY_DIR@") set(src_dir "@PROJECT_SOURCE_DIR@") diff --git a/src/libopentld/CMakeLists.txt b/src/libopentld/CMakeLists.txt index 740c07d2..b9aa139f 100644 --- a/src/libopentld/CMakeLists.txt +++ b/src/libopentld/CMakeLists.txt @@ -3,8 +3,10 @@ include_directories(imacq mftracker tld ../3rdparty/cvblobs - ${OpenCV_INCLUDE_DIRS}) -link_directories(${OpenCV_LIB_DIR}) + ${OpenCV_INCLUDE_DIRS} + ${OpenCL_INCLUDE_DIRS}) +link_directories(${OpenCV_LIB_DIR} +${OpenCL_LIB_DIR}) add_library(libopentld imacq/ImAcq.cpp @@ -42,7 +44,7 @@ add_library(libopentld tld/TLDUtil.h tld/VarianceFilter.h) -target_link_libraries(libopentld ${OpenCV_LIBS}) +target_link_libraries(libopentld ${OpenCV_LIBS} ${OPENCL_LIBRARIES}) set_target_properties(libopentld PROPERTIES OUTPUT_NAME opentld) diff --git a/src/opentld/CMakeLists.txt b/src/opentld/CMakeLists.txt index 23170b33..dd20a37d 100644 --- a/src/opentld/CMakeLists.txt +++ b/src/opentld/CMakeLists.txt @@ -1,12 +1,14 @@ -link_directories(${OpenCV_LIB_DIR}) +link_directories(${OpenCV_LIB_DIR} +${OpenCL_LIB_DIR}) include_directories(main ../libopentld/imacq ../libopentld/mftracker ../libopentld/tld ../3rdparty/cvblobs - ${OpenCV_INCLUDE_DIRS}) + ${OpenCV_INCLUDE_DIRS} + ${OpenCL_INCLUDE_DIRS}) if(NOT USE_SYSTEM_LIBS) include_directories(../3rdparty/libconfig) @@ -26,14 +28,14 @@ add_library(main main/Settings.h main/Trajectory.h) -target_link_libraries(main libopentld cvblobs config++ ${OpenCV_LIBS}) +target_link_libraries(main libopentld cvblobs config++ ${OpenCV_LIBS} ${OPENCL_LIBRARIES}) #------------------------------------------------------------------------------- # opentld add_executable(opentld OpenTLD.cpp) -target_link_libraries(opentld main libopentld cvblobs config++ ${OpenCV_LIBS}) +target_link_libraries(opentld main libopentld cvblobs config++ ${OpenCV_LIBS} ${OPENCL_LIBRARIES}) install(TARGETS opentld DESTINATION bin) @@ -57,7 +59,7 @@ if(BUILD_QOPENTLD) set(QOPENTLD_SOURCES ${QOPENTLD_SOURCES} ${QOPENTLD_RCS_CPP} ${QOPENTLD_HEADERS_MOC} ${QOPENTLD_FORMS_HEADERS}) add_executable(qopentld ${QOPENTLD_SOURCES}) - target_link_libraries(qopentld main libopentld cvblobs config++ ${OpenCV_LIBS} ${QT_LIBRARIES}) + target_link_libraries(qopentld main libopentld cvblobs config++ ${OpenCV_LIBS ${OPENCL_LIBRARIES}} ${QT_LIBRARIES}) install(TARGETS qopentld DESTINATION bin) endif(BUILD_QOPENTLD) From dc8162908277f1a12a4b0b7f63908901ee6ec590 Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Tue, 12 Sep 2017 17:02:51 +0800 Subject: [PATCH 08/14] Prepare for NNclassifier filter Prepare for NNclassifier filter --- src/libopentld/tld/DetectorCascade.cpp | 8 ++++++++ src/libopentld/tld/EnsembleClassifier.cpp | 12 +++++++++--- src/libopentld/tld/TLD.cpp | 4 ++-- src/libopentld/tld/switch.h | 3 ++- src/opentld/main/Main.cpp | 2 +- 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/libopentld/tld/DetectorCascade.cpp b/src/libopentld/tld/DetectorCascade.cpp index eaaa869e..7effc099 100644 --- a/src/libopentld/tld/DetectorCascade.cpp +++ b/src/libopentld/tld/DetectorCascade.cpp @@ -506,6 +506,7 @@ namespace tld //} //printf("detectionResult[753666]=%f\n",detectionResult->posteriors[753666]); +#ifndef clNNClassifier for (int i = 0, count = 0; i < numWindows; i++) { // printf("I am victor %d\n", i); @@ -526,6 +527,13 @@ namespace tld } } + +#else + + +#endif + + //printf("confident_size %d\n\n", detectionResult->confidentIndices->size());//zhaodc //Cluster diff --git a/src/libopentld/tld/EnsembleClassifier.cpp b/src/libopentld/tld/EnsembleClassifier.cpp index c9c3591f..d06c943f 100644 --- a/src/libopentld/tld/EnsembleClassifier.cpp +++ b/src/libopentld/tld/EnsembleClassifier.cpp @@ -633,10 +633,13 @@ namespace tld t_sqsum.release(); clReleaseMemObject(tsumdata); clReleaseMemObject(tsqsumdata); - /******* ******/ - /*End Integral */ - /****** *******/ +/* + * End Integral + */ +/* + * Begain ensemble filter or Posterior + */ if (!enabled) return true; int tld_window_offset_size = TLD_WINDOW_OFFSET_SIZE; @@ -703,6 +706,9 @@ namespace tld for (int i = 0; i < 6; i++) status = clReleaseEvent(events[i]); //cout << "o" << endl; +/* +* End ensemble filter or Posterior +*/ clReleaseMemObject(oclbuffWindowsOffset); clReleaseMemObject(oclbufffeatureOffsets); diff --git a/src/libopentld/tld/TLD.cpp b/src/libopentld/tld/TLD.cpp index fcaf02f7..4343a358 100644 --- a/src/libopentld/tld/TLD.cpp +++ b/src/libopentld/tld/TLD.cpp @@ -499,7 +499,7 @@ namespace tld float *overlap = new float[detectorCascade->numWindows]; // double tic = cvGetTickCount(); -#if cloverlap +#if clOverlap cltldOverlapRect_self(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); #else tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); @@ -626,7 +626,7 @@ namespace tld #if PrintTime_overlap double tic = cvGetTickCount(); #endif -#if cloverlap +#if clOverlap cltldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); #else tldOverlapRect(detectorCascade->windows, detectorCascade->numWindows, currBB, overlap); diff --git a/src/libopentld/tld/switch.h b/src/libopentld/tld/switch.h index ae7f5446..98166751 100644 --- a/src/libopentld/tld/switch.h +++ b/src/libopentld/tld/switch.h @@ -4,7 +4,8 @@ #define clIntegral 1 #define clIntegral_m 1 #define clIntegral_m_1 0 -#define cloverlap 0 +#define clOverlap 0 +#define clNNClassifier 0 #define PrintTime_overlap 0 #define PrintTime_detect 0 diff --git a/src/opentld/main/Main.cpp b/src/opentld/main/Main.cpp index b57ad4c6..bba5dcd1 100644 --- a/src/opentld/main/Main.cpp +++ b/src/opentld/main/Main.cpp @@ -51,7 +51,7 @@ void Main::doWork() trajectory.init(trajectoryLength); } - if (selectManually = 1) + if (selectManually) { CvRect box; From 2a61ba1b4c1360a50ce4ac50e3fa883acca01a8e Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Wed, 13 Sep 2017 17:00:01 +0800 Subject: [PATCH 09/14] Add some codes for NNClassify using OpenCL Add some codes for NNClassify using OpenCL --- .../EnsembleClassifierFilter_kernel.cl.cpp | 18 +++ src/libopentld/tld/DetectorCascade.cpp | 43 +++++- src/libopentld/tld/DetectorCascade.h | 1 + src/libopentld/tld/EnsembleClassifier.cpp | 2 +- src/libopentld/tld/NNClassifier.cpp | 142 +++++++++++++++++- src/libopentld/tld/NNClassifier.h | 57 +++++++ src/libopentld/tld/switch.h | 2 +- 7 files changed, 258 insertions(+), 7 deletions(-) diff --git a/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp index 10a45bd7..8d0a4da5 100644 --- a/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp +++ b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp @@ -366,3 +366,21 @@ kernel void integral_rows_DD(__global int4 *srcsum,__global float4 * srcsqsum,__ // if(lid==0 && gid==0) // printf("sum=%d,sqsum=%f\n",sum[lid],sqsum[lid] ); } + + + + +kernel void nnClassifier( + __global int* oclbuffWindowsOffset, + __global int* oclbuffII, + __global float* oclbuffIISqure, + __global float* oclbuffDetectionResultVarious, + __global float* oclbuffDetectionResultPosteriors, + const unsigned int numWindows, + const float minVar, + __global int* windowFlags) +{ + + + +} \ No newline at end of file diff --git a/src/libopentld/tld/DetectorCascade.cpp b/src/libopentld/tld/DetectorCascade.cpp index 7effc099..2870df60 100644 --- a/src/libopentld/tld/DetectorCascade.cpp +++ b/src/libopentld/tld/DetectorCascade.cpp @@ -158,6 +158,7 @@ namespace tld ensembleClassifier->numFeatures = numFeatures; ensembleClassifier->numTrees = numTrees; nnClassifier->windows = windows; + nnClassifier->numWindows = numWindows; clustering->windows = windows; clustering->numWindows = numWindows; @@ -195,6 +196,13 @@ namespace tld ensembleClassifier->kernel_intgegral_rows_en = kernel_intgegral_rows; //kernel_variance = clCreateKernel(program, "varianceFilter", NULL); + //nnClassifier + nnClassifier->platform = platform; + nnClassifier->devices = devices; + nnClassifier->context = context; + nnClassifier->commandQueue = commandQueue; + nnClassifier->program = program; + nnClassifier->kernel_nnClassifier = kernel_nnClassifier; @@ -414,6 +422,7 @@ namespace tld kernel_ensemble = clCreateKernel(program, "EnsembleClassifierFilter", NULL); kernel_intgegral_cols = clCreateKernel(program, "integral_cols_DD", NULL); kernel_intgegral_rows = clCreateKernel(program, "integral_rows_DD", NULL); + kernel_nnClassifier = clCreateKernel(program, "nnClassifier", NULL); } void DetectorCascade::oclRelease() @@ -506,7 +515,34 @@ namespace tld //} //printf("detectionResult[753666]=%f\n",detectionResult->posteriors[753666]); -#ifndef clNNClassifier +#if clNNClassifier + for (int i = 0, count = 0; i < numWindows; i++) + { + if (detectionResult->posteriors[i] >= 0.5f) + { + //float values; + //int index; + //bool flag; + NNClassifier->nnClassifyStructInstance.conf = 0.0f; + NNClassifier->nnClassifyStructInstance.index = i; + NNClassifier->nnClassifyStructInstance.flag = false; + nnClassifier->candidatesToNNClassify->push_back(NNClassifier->nnClassifyStructInstance); + + nnClassifier->candidatesToNNClassifyIndexArray->push_back(i); + } + } + + nnClassifier->clNNFilter(img); + + //v.clear(); + //v.shrink_to_fit(); + + nnClassifier->candidatesToNNClassify->clear(); + + nnClassifier->candidatesToNNClassifyIndexArray->clear(); + + +#else for (int i = 0, count = 0; i < numWindows; i++) { // printf("I am victor %d\n", i); @@ -526,14 +562,13 @@ namespace tld } - } + } -#else - #endif + //printf("confident_size %d\n\n", detectionResult->confidentIndices->size());//zhaodc //Cluster diff --git a/src/libopentld/tld/DetectorCascade.h b/src/libopentld/tld/DetectorCascade.h index cd3f2ea4..d8fa8242 100644 --- a/src/libopentld/tld/DetectorCascade.h +++ b/src/libopentld/tld/DetectorCascade.h @@ -93,6 +93,7 @@ namespace tld cl_kernel kernel_variance; cl_kernel kernel_intgegral_cols; cl_kernel kernel_intgegral_rows; + cl_kernel kernel_nnClassifier; int oclSetup(); int oclBuildKernel(const char *kernelName); diff --git a/src/libopentld/tld/EnsembleClassifier.cpp b/src/libopentld/tld/EnsembleClassifier.cpp index d06c943f..d9c8dc4c 100644 --- a/src/libopentld/tld/EnsembleClassifier.cpp +++ b/src/libopentld/tld/EnsembleClassifier.cpp @@ -638,7 +638,7 @@ namespace tld */ /* - * Begain ensemble filter or Posterior + * Begain variance && ensemble filter or Posterior */ if (!enabled) return true; diff --git a/src/libopentld/tld/NNClassifier.cpp b/src/libopentld/tld/NNClassifier.cpp index 5c6c313a..a6d4d29b 100644 --- a/src/libopentld/tld/NNClassifier.cpp +++ b/src/libopentld/tld/NNClassifier.cpp @@ -41,7 +41,10 @@ namespace tld truePositives = new vector(); falsePositives = new vector(); - + candidatesToNNClassify = new vector(); // another method is to array + candidatesToNNClassifyIndexArray = new vector(); + clNNResultsArray = NULL; + kernel_nnClassifier = NULL; } NNClassifier::~NNClassifier() @@ -50,12 +53,14 @@ namespace tld delete truePositives; delete falsePositives; + delete clNNResultsArray; } void NNClassifier::release() { falsePositives->clear(); truePositives->clear(); + candidatesToNNClassify->clear(); } float NNClassifier::ncc(float *f1, float *f2) @@ -161,6 +166,141 @@ namespace tld return true; } + + bool NNClassifier::clNNFilter(const cv::Mat &img) + { + cl_event events[1]; + + int truePostiveSize = truePositives->size(); + int falsePositiveSize = falsePositives->size(); + clNNResultsArray = new float[ truePostiveSize * falsePositiveSize]; + printf("..................\n"); + printf("truePositives->size()=%d \t,falsePositives->size()=%d\n", truePositives->size(), falsePositives->size()); + printf("..................\n"); + + // case one + if (truePositives->empty()) + { + //--return 0; + //++set all ccorr_max_p equale to 1.0; + for (int i = 0, count = 0; i < this->candidatesToNNClassify->size(); i++) + candidatesToNNClassify->at(i).conf = 0.0f; + goto EndofFuction; + } + // case two + if (!truePositives->empty() && falsePositives->empty()) + { + //++ set all ccorr_max_p equale to 1.0; + for (int i = 0, count = 0; i < this->candidatesToNNClassify->size(); i++) + candidatesToNNClassify->at(i).conf = 1.0f; + goto EndofFuction; + + + } + //case three + if (!truePositives->empty() && !falsePositives->empty()) + { + int tld_window_size = TLD_WINDOW_SIZE; + + oclbuffWindows = clCreateBuffer(context, CL_MEM_READ_ONLY, TLD_WINDOW_SIZE * numWindows *sizeof(int), NULL, NULL); + + oclSrcData = clCreateBuffer(context, CL_MEM_READ_ONLY, (img.cols) * (img.rows) * sizeof(uchar), NULL, NULL); + + + + + + + /* copy from */ + + cl_mem windows = clCreateBuffer(context, CL_MEM_READ_ONLY, TLD_WINDOW_SIZE * numWindows *sizeof(int), NULL, NULL); + + + + cl_mem oclbuffWindowsOffset = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (TLD_WINDOW_OFFSET_SIZE * numWindows) * sizeof(int), (void *)windowOffsets, NULL); + cl_mem oclbuffII = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (img.size().width)*(img.size().height) * sizeof(int), (void *)iSumMat.data, NULL); + cl_mem oclbuffIISqure = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (img.size().width)*(img.size().height) * sizeof(int), (void *)fSqreSumMat.data, NULL); + cl_mem oclbuffDetectionResultVarious = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)detectionResult->variances, NULL); + cl_mem oclbuffDetectionResultPosteriors = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)detectionResult->posteriors, NULL); + + cl_mem oclbuffDetectionwindowFlags = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(int), (void *)detectionResult->windowFlags, NULL); + + /*Step 8: Create kernel object */ + /*Step 9: Sets Kernel arguments.*/ + status = clSetKernelArg(kernel_nnClassifier, 0, sizeof(cl_mem), (void *)&oclbuffWindowsOffset); + status = clSetKernelArg(kernel_nnClassifier, 1, sizeof(cl_mem), (void *)&oclbuffII); + status = clSetKernelArg(kernel_nnClassifier, 2, sizeof(cl_mem), (void *)&oclbuffIISqure); + status = clSetKernelArg(kernel_nnClassifier, 3, sizeof(cl_mem), (void *)&oclbuffDetectionResultVarious); + status = clSetKernelArg(kernel_nnClassifier, 4, sizeof(cl_mem), (void *)&oclbuffDetectionResultPosteriors); + status = clSetKernelArg(kernel_nnClassifier, 5, sizeof(int), (void *)&numWindows); + status = clSetKernelArg(kernel_nnClassifier, 6, sizeof(float), (void *)&minVar); + status = clSetKernelArg(kernel_nnClassifier, 7, sizeof(cl_mem), (void*)&oclbuffDetectionwindowFlags); + //status = clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *)&TLD_WINDOW_OFFSET_SIZE); + + /*Step 10: Running the kernel.*/ + printf("begore opencl kernel numWindows=%d\n", numWindows); + size_t global_work_size[1] = { truePostiveSize*falsePositiveSize }; + size_t local_work_size[1] = { 256 }; + status = clEnqueueNDRangeKernel(commandQueue, kernel_nnClassifier, 1, NULL, global_work_size, NULL, 0, NULL, &events[0]); + if (status != CL_SUCCESS) + { + cout << "Error:VarianceFilter EnqueueNDRangeKernel!" << endl; + //return false; + } + status = clWaitForEvents(1, &events[0]); + + if (status != CL_SUCCESS) + { + printf("Error: Waiting for kernel run to finish. (clWaitForEvents0)\n"); + + } + //cout << "o" << endl; + + status = clReleaseEvent(events[0]); + + //printf("end using GPU*************************\n"); + //for (int i = 0; i < numWindows; i++) + // if(detectionResult->windowFlags[i]==1) + // printf("detectionResult[%d] is %d\n", i,detectionResult->windowFlags[i]); + + /*Step 11: Read the cout put back to host memory.*/ + //status = clEnqueueReadBuffer(commandQueue, outputBuffer, CL_TRUE, 0, 12 * sizeof(char), output, 0, NULL, NULL); + + + + clReleaseKernel(kernel_nnClassifier); + clReleaseProgram(program); + + clReleaseMemObject(oclbuffWindowsOffset); + clReleaseMemObject(oclbuffII); + clReleaseMemObject(oclbuffIISqure); + clReleaseMemObject(oclbuffDetectionResultVarious); + clReleaseMemObject(oclbuffDetectionResultPosteriors); + clReleaseMemObject(oclbuffDetectionwindowFlags); + return true; + + + /*end copy */ + + + + } + + + + + + + + + + float ccorr_max_p = 0; + EndofFuction: + return true; + + + } + void NNClassifier::learn(vector patches) { //TODO: Randomization might be a good idea here diff --git a/src/libopentld/tld/NNClassifier.h b/src/libopentld/tld/NNClassifier.h index 662c6edf..20800b2e 100644 --- a/src/libopentld/tld/NNClassifier.h +++ b/src/libopentld/tld/NNClassifier.h @@ -29,6 +29,7 @@ #include #include +#include #include "NormalizedPatch.h" #include "DetectionResult.h" @@ -36,19 +37,39 @@ namespace tld { + +class nnClassifyStruct +{ +public: +float conf; +int index; +bool flag; +}; + + class NNClassifier { float ncc(float *f1, float *f2); + public: bool enabled; int *windows; + //Configurable members + + int numWindows; float thetaFP; float thetaTP; DetectionResult *detectionResult; std::vector* falsePositives; std::vector* truePositives; + float *clNNResultsArray; + std::vector *candidatesToNNClassifyIndexArray; + std::vector * candidatesToNNClassify; + + nnClassifyStruct nnClassifyStructInstance; + NNClassifier(); virtual ~NNClassifier(); @@ -58,6 +79,42 @@ class NNClassifier float classifyWindow(const cv::Mat &img, int windowIdx); void learn(std::vector patches); bool filter(const cv::Mat &img, int windowIdx); + + + + //for opencl begin in NNclassifier + bool clNNFilter(const cv::Mat &img); + // variable members + cl_platform_id platform; //the chosen platform + cl_int status; + cl_device_id *devices; + cl_context context; + cl_command_queue commandQueue; + cl_program program; + cl_mem inputBuffer; + cl_mem outputBuffer; + cl_kernel variance_ensemble_kernel; + cl_kernel kernel_intgegral_cols_en, kernel_intgegral_rows_en; + + cl_mem oclSrcData; + cl_mem oclbuffWindows; + + cl_mem oclbuffWindowsOffset; + cl_mem oclbufffeatureOffsets; + cl_mem oclbuffDetectionResultfeatureVectors; + cl_mem oclbuffDetectionResultPosteriors; + cl_mem oclbuffDetectionwindowFlags; + cl_mem oclbuffPosteriors; + cl_mem oclbuffDetectionResultVarious; + cl_mem oclbuffImgData; + cl_mem oclbuffII; + cl_mem oclbuffIISqure; + size_t local_work_size[1]; + cl_kernel kernel_nnClassifier; + + + //for opencl end in NNclassifier + }; } /* namespace tld */ diff --git a/src/libopentld/tld/switch.h b/src/libopentld/tld/switch.h index 98166751..9ed8667a 100644 --- a/src/libopentld/tld/switch.h +++ b/src/libopentld/tld/switch.h @@ -5,7 +5,7 @@ #define clIntegral_m 1 #define clIntegral_m_1 0 #define clOverlap 0 -#define clNNClassifier 0 +#define clNNClassifier 0 #define PrintTime_overlap 0 #define PrintTime_detect 0 From dbf8ce1db05b40fd8e5200126f5f6f2119104480 Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Thu, 14 Sep 2017 17:50:56 +0800 Subject: [PATCH 10/14] add NNClassifier->clNNFilter completed NNClassifier->clNNFilter interface, will do conferenced opencl kernel. --- .../EnsembleClassifierFilter_kernel.cl.cpp | 51 ++++-- src/libopentld/tld/DetectorCascade.cpp | 100 ++++++++++- src/libopentld/tld/NNClassifier.cpp | 168 +++++++++++++----- src/libopentld/tld/NNClassifier.h | 50 +++--- 4 files changed, 280 insertions(+), 89 deletions(-) diff --git a/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp index 8d0a4da5..a173a64d 100644 --- a/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp +++ b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp @@ -4,6 +4,43 @@ + +kernel void nnClassifier( + __global uchar * img, + __global int* windows, + __global int* WindowIndexArray, + __global float *NNResultsArray, + const unsigned int tld_window_size, + const unsigned int truePostiveSize, + const unsigned int falsePositiveSize, + __global float * truePostiveData, + __global float * FalsePostiveData) +{ + + int positiveID = get_global_id(0); //positiveID is one of truePostiveSize or falsePositiveSize + if (positiveID < truePostiveSize) + { + + //TODO + + + } + // tld_window_size = 5 + int bbox0 = windows[positiveID * 5 + 0]; //x + int bbox1 = windows[positiveID * 5 + 1]; // y + int bbox2 = windows[positiveID * 5 + 2]; // width + int bbox3 = windows[positiveID * 5 + 3]; //height + int bbox4 = windows[positiveID * 5 + 4]; + float patchValue[225]; + //resize(image(source(x, y, width, height), 15 * 15, patchValue); //opencv resize function with bilinear method. + + + + + +} + + __kernel void EnsembleClassifierFilter( const int numIndices, const unsigned int numTrees, @@ -370,17 +407,3 @@ kernel void integral_rows_DD(__global int4 *srcsum,__global float4 * srcsqsum,__ -kernel void nnClassifier( - __global int* oclbuffWindowsOffset, - __global int* oclbuffII, - __global float* oclbuffIISqure, - __global float* oclbuffDetectionResultVarious, - __global float* oclbuffDetectionResultPosteriors, - const unsigned int numWindows, - const float minVar, - __global int* windowFlags) -{ - - - -} \ No newline at end of file diff --git a/src/libopentld/tld/DetectorCascade.cpp b/src/libopentld/tld/DetectorCascade.cpp index 2870df60..86f63b9a 100644 --- a/src/libopentld/tld/DetectorCascade.cpp +++ b/src/libopentld/tld/DetectorCascade.cpp @@ -523,24 +523,34 @@ namespace tld //float values; //int index; //bool flag; - NNClassifier->nnClassifyStructInstance.conf = 0.0f; - NNClassifier->nnClassifyStructInstance.index = i; - NNClassifier->nnClassifyStructInstance.flag = false; - nnClassifier->candidatesToNNClassify->push_back(NNClassifier->nnClassifyStructInstance); + nnClassifier->nnClassifyStructInstance.conf = 0.0f; + nnClassifier->nnClassifyStructInstance.index = i; + nnClassifier->nnClassifyStructInstance.flag = false; + nnClassifier->candidatesToNNClassifyVector->push_back(nnClassifier->nnClassifyStructInstance); - nnClassifier->candidatesToNNClassifyIndexArray->push_back(i); + nnClassifier->candidatesToNNClassifyIndexVector->push_back(i); } + } nnClassifier->clNNFilter(img); + + + for (int i = 0; i < nnClassifier->candidatesToNNClassifyVector->size(); i++) + { + if (nnClassifier->candidatesToNNClassifyVector->at(i).conf > nnClassifier->thetaTP) + detectionResult->confidentIndices->push_back(nnClassifier->candidatesToNNClassifyVector->at(i).index); + } //v.clear(); //v.shrink_to_fit(); - nnClassifier->candidatesToNNClassify->clear(); - nnClassifier->candidatesToNNClassifyIndexArray->clear(); + //·µ»Ø + nnClassifier->candidatesToNNClassifyIndexVector->clear(); + nnClassifier->candidatesToNNClassifyVector->clear(); + #else for (int i = 0, count = 0; i < numWindows; i++) @@ -574,6 +584,82 @@ namespace tld //Cluster clustering->clusterConfidentIndices(); + detectionResult->containsValidData = true; + + EndofCldetect: + ; + +} + void DetectorCascade::detect(const Mat &img) + { + //For every bounding box, the output is confidence, pattern, variance + + detectionResult->reset(); + + if (!initialised) + { + return; + } + + //Prepare components + foregroundDetector->nextIteration(img); //Calculates foreground + varianceFilter->nextIteration(img); //Calculates integral images + ensembleClassifier->nextIteration(img); + +#pragma omp parallel for + + for (int i = 0; i < numWindows; i++) + { + + int *window = &windows[TLD_WINDOW_SIZE * i]; + + if (foregroundDetector->isActive()) + { + bool isInside = false; + + for (size_t j = 0; j < detectionResult->fgList->size(); j++) + { + + int bgBox[4]; + tldRectToArray(detectionResult->fgList->at(j), bgBox); + + if (tldIsInside(window, bgBox)) //TODO: This is inefficient and should be replaced by a quadtree + { + isInside = true; + } + } + + if (!isInside) + { + detectionResult->posteriors[i] = 0; + continue; + } + } + + if (!varianceFilter->filter(i)) + { + detectionResult->posteriors[i] = 0; + continue; + } + + if (!ensembleClassifier->filter(i)) + { + continue; + } + + if (!nnClassifier->filter(img, i)) + { + continue; + } + + detectionResult->confidentIndices->push_back(i); + + + } + + //Cluster + clustering->clusterConfidentIndices(); + detectionResult->containsValidData = true; } diff --git a/src/libopentld/tld/NNClassifier.cpp b/src/libopentld/tld/NNClassifier.cpp index a6d4d29b..067d4380 100644 --- a/src/libopentld/tld/NNClassifier.cpp +++ b/src/libopentld/tld/NNClassifier.cpp @@ -34,6 +34,9 @@ using namespace cv; namespace tld { + + + NNClassifier::NNClassifier() { thetaFP = .5; @@ -41,10 +44,14 @@ namespace tld truePositives = new vector(); falsePositives = new vector(); - candidatesToNNClassify = new vector(); // another method is to array - candidatesToNNClassifyIndexArray = new vector(); - clNNResultsArray = NULL; + candidatesToNNClassifyVector = new vector(); // another method is to array + candidatesToNNClassifyIndexVector = new vector(); + pcandidatesToNNClassifyIndexArray = NULL; + pNNResultsArray = NULL; kernel_nnClassifier = NULL; + pSrcTruePostiveData = NULL; + pSrcFalsePostiveData = NULL; + //nnClassifyStructInstance = new nnClassifyStruct; } NNClassifier::~NNClassifier() @@ -53,16 +60,49 @@ namespace tld delete truePositives; delete falsePositives; - delete clNNResultsArray; + delete pNNResultsArray; + delete pcandidatesToNNClassifyIndexArray; } void NNClassifier::release() { falsePositives->clear(); truePositives->clear(); - candidatesToNNClassify->clear(); + candidatesToNNClassifyVector->clear(); + candidatesToNNClassifyIndexVector->clear(); } + void NNClassifier::VectorToArray() + { + + int truePostiveSize = truePositives->size(); + int falsePositiveSize = falsePositives->size(); + + pSrcTruePostiveData = new float[truePostiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE]; + float *p = pSrcTruePostiveData; + float *q; + for (int i = 0; i < truePostiveSize; i++) + { + + q = truePositives->at(i).values; + for (int j = 0; j < TLD_PATCH_SIZE *TLD_PATCH_SIZE; j++) + *p++ = *q++; + + } + + pSrcFalsePostiveData = new float[falsePositiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE]; + p = pSrcTruePostiveData; + for (int i = 0; i < truePostiveSize; i++) + { + + q = truePositives->at(i).values; + for (int j = 0; j < TLD_PATCH_SIZE *TLD_PATCH_SIZE; j++) + *p++ = *q++; + } + + + + } float NNClassifier::ncc(float *f1, float *f2) { double corr = 0; @@ -169,11 +209,22 @@ namespace tld bool NNClassifier::clNNFilter(const cv::Mat &img) { + cl_event events[1]; int truePostiveSize = truePositives->size(); int falsePositiveSize = falsePositives->size(); - clNNResultsArray = new float[ truePostiveSize * falsePositiveSize]; + + if (0 == candidatesToNNClassifyIndexVector->size()) + return false; + pNNResultsArray = new float[ (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size()]; + + pcandidatesToNNClassifyIndexArray = new float[candidatesToNNClassifyIndexVector->size()]; + float *p = pcandidatesToNNClassifyIndexArray; + + for (int i = 0; i < candidatesToNNClassifyIndexVector->size(); i++) + *p++ = candidatesToNNClassifyIndexVector->at(i); + printf("..................\n"); printf("truePositives->size()=%d \t,falsePositives->size()=%d\n", truePositives->size(), falsePositives->size()); printf("..................\n"); @@ -183,63 +234,84 @@ namespace tld { //--return 0; //++set all ccorr_max_p equale to 1.0; - for (int i = 0, count = 0; i < this->candidatesToNNClassify->size(); i++) - candidatesToNNClassify->at(i).conf = 0.0f; + for (int i = 0, count = 0; i < this->candidatesToNNClassifyVector->size(); i++) + { + candidatesToNNClassifyVector->at(i).conf = 0.0f; + candidatesToNNClassifyVector->at(i).index = i; + candidatesToNNClassifyVector->at(i).flag = false; + + } goto EndofFuction; } // case two if (!truePositives->empty() && falsePositives->empty()) { //++ set all ccorr_max_p equale to 1.0; - for (int i = 0, count = 0; i < this->candidatesToNNClassify->size(); i++) - candidatesToNNClassify->at(i).conf = 1.0f; - goto EndofFuction; - + for (int i = 0, count = 0; i < this->candidatesToNNClassifyVector->size(); i++) + { + candidatesToNNClassifyVector->at(i).conf = 1.0f; + candidatesToNNClassifyVector->at(i).index = i; + candidatesToNNClassifyVector->at(i).flag = true; + } + + goto EndofFuction; } //case three if (!truePositives->empty() && !falsePositives->empty()) { + int tld_window_size = TLD_WINDOW_SIZE; - - oclbuffWindows = clCreateBuffer(context, CL_MEM_READ_ONLY, TLD_WINDOW_SIZE * numWindows *sizeof(int), NULL, NULL); - - oclSrcData = clCreateBuffer(context, CL_MEM_READ_ONLY, (img.cols) * (img.rows) * sizeof(uchar), NULL, NULL); - - - - - - - /* copy from */ + //VectorToArray(); + // Begin Vector to Array + pSrcTruePostiveData = new float[truePostiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE]; + float *p = pSrcTruePostiveData; + float *q; + for (int i = 0; i < truePostiveSize; i++) + { + q = truePositives->at(i).values; + for (int j = 0; j < TLD_PATCH_SIZE *TLD_PATCH_SIZE; j++) + *p++ = *q++; + } + pSrcFalsePostiveData = new float[falsePositiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE]; + p = pSrcFalsePostiveData; + for (int i = 0; i < truePostiveSize; i++) + { + q = falsePositives->at(i).values; + for (int j = 0; j < TLD_PATCH_SIZE *TLD_PATCH_SIZE; j++) + *p++ = *q++; + } + // End Vector to Array + oclbufferWindows = clCreateBuffer(context, CL_MEM_READ_ONLY, TLD_WINDOW_SIZE * numWindows *sizeof(int), (void*) windows, NULL); + oclbufferSrcData = clCreateBuffer(context, CL_MEM_READ_ONLY, (img.cols) * (img.rows) * sizeof(uchar), (void*)img.data, NULL); + oclbuffercandidatesToNNClassifyIndexArray = clCreateBuffer(context, CL_MEM_READ_ONLY, candidatesToNNClassifyVector->size() * sizeof(int), (void*)pcandidatesToNNClassifyIndexArray, NULL); + oclbufferpNNResultsArray = clCreateBuffer(context, CL_MEM_READ_ONLY, (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size()* sizeof(float), (void*)pNNResultsArray, NULL); - cl_mem windows = clCreateBuffer(context, CL_MEM_READ_ONLY, TLD_WINDOW_SIZE * numWindows *sizeof(int), NULL, NULL); + oclbufferSrcTruePostiveData = clCreateBuffer(context, CL_MEM_READ_ONLY, truePostiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE* sizeof(float), (void*)pSrcTruePostiveData, NULL); + oclbufferSrcFalsePostiveData = clCreateBuffer(context, CL_MEM_READ_ONLY, falsePositiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE* sizeof(float), (void*)pSrcFalsePostiveData, NULL); - cl_mem oclbuffWindowsOffset = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (TLD_WINDOW_OFFSET_SIZE * numWindows) * sizeof(int), (void *)windowOffsets, NULL); - cl_mem oclbuffII = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (img.size().width)*(img.size().height) * sizeof(int), (void *)iSumMat.data, NULL); - cl_mem oclbuffIISqure = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, (img.size().width)*(img.size().height) * sizeof(int), (void *)fSqreSumMat.data, NULL); - cl_mem oclbuffDetectionResultVarious = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)detectionResult->variances, NULL); - cl_mem oclbuffDetectionResultPosteriors = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(float), (void *)detectionResult->posteriors, NULL); - cl_mem oclbuffDetectionwindowFlags = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, (numWindows)* sizeof(int), (void *)detectionResult->windowFlags, NULL); + // /*Step 8: Create kernel object */ /*Step 9: Sets Kernel arguments.*/ - status = clSetKernelArg(kernel_nnClassifier, 0, sizeof(cl_mem), (void *)&oclbuffWindowsOffset); - status = clSetKernelArg(kernel_nnClassifier, 1, sizeof(cl_mem), (void *)&oclbuffII); - status = clSetKernelArg(kernel_nnClassifier, 2, sizeof(cl_mem), (void *)&oclbuffIISqure); - status = clSetKernelArg(kernel_nnClassifier, 3, sizeof(cl_mem), (void *)&oclbuffDetectionResultVarious); - status = clSetKernelArg(kernel_nnClassifier, 4, sizeof(cl_mem), (void *)&oclbuffDetectionResultPosteriors); - status = clSetKernelArg(kernel_nnClassifier, 5, sizeof(int), (void *)&numWindows); - status = clSetKernelArg(kernel_nnClassifier, 6, sizeof(float), (void *)&minVar); - status = clSetKernelArg(kernel_nnClassifier, 7, sizeof(cl_mem), (void*)&oclbuffDetectionwindowFlags); + status = clSetKernelArg(kernel_nnClassifier, 0, sizeof(cl_mem), (void *)&oclbufferSrcData); + status = clSetKernelArg(kernel_nnClassifier, 1, sizeof(cl_mem), (void *)&oclbufferWindows); + status = clSetKernelArg(kernel_nnClassifier, 2, sizeof(cl_mem), (void *)&oclbuffercandidatesToNNClassifyIndexArray); + status = clSetKernelArg(kernel_nnClassifier, 3, sizeof(cl_mem), (void *)&oclbufferpNNResultsArray); + status = clSetKernelArg(kernel_nnClassifier, 4, sizeof(int), (void *)&tld_window_size); + status = clSetKernelArg(kernel_nnClassifier, 5, sizeof(int), (void *)&truePostiveSize); + status = clSetKernelArg(kernel_nnClassifier, 6, sizeof(int), (void *)&falsePositiveSize); + status = clSetKernelArg(kernel_nnClassifier, 7, sizeof(cl_mem), (void *)&oclbufferSrcTruePostiveData); + status = clSetKernelArg(kernel_nnClassifier, 8, sizeof(cl_mem), (void *)&oclbufferSrcFalsePostiveData); + //status = clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *)&TLD_WINDOW_OFFSET_SIZE); /*Step 10: Running the kernel.*/ printf("begore opencl kernel numWindows=%d\n", numWindows); - size_t global_work_size[1] = { truePostiveSize*falsePositiveSize }; + size_t global_work_size[1] = { truePostiveSize+falsePositiveSize }; size_t local_work_size[1] = { 256 }; status = clEnqueueNDRangeKernel(commandQueue, kernel_nnClassifier, 1, NULL, global_work_size, NULL, 0, NULL, &events[0]); if (status != CL_SUCCESS) @@ -267,16 +339,22 @@ namespace tld //status = clEnqueueReadBuffer(commandQueue, outputBuffer, CL_TRUE, 0, 12 * sizeof(char), output, 0, NULL, NULL); + delete pSrcTruePostiveData; + pSrcTruePostiveData = NULL; + delete pSrcFalsePostiveData; + pSrcFalsePostiveData = NULL; + + clReleaseKernel(kernel_nnClassifier); clReleaseProgram(program); + clReleaseMemObject(oclbufferSrcData); + clReleaseMemObject(oclbufferWindows); + clReleaseMemObject(oclbuffercandidatesToNNClassifyIndexArray); + clReleaseMemObject(oclbufferpNNResultsArray); + + - clReleaseMemObject(oclbuffWindowsOffset); - clReleaseMemObject(oclbuffII); - clReleaseMemObject(oclbuffIISqure); - clReleaseMemObject(oclbuffDetectionResultVarious); - clReleaseMemObject(oclbuffDetectionResultPosteriors); - clReleaseMemObject(oclbuffDetectionwindowFlags); return true; diff --git a/src/libopentld/tld/NNClassifier.h b/src/libopentld/tld/NNClassifier.h index 20800b2e..7ffacf8d 100644 --- a/src/libopentld/tld/NNClassifier.h +++ b/src/libopentld/tld/NNClassifier.h @@ -62,10 +62,11 @@ class NNClassifier float thetaTP; DetectionResult *detectionResult; std::vector* falsePositives; - std::vector* truePositives; - float *clNNResultsArray; - std::vector *candidatesToNNClassifyIndexArray; - std::vector * candidatesToNNClassify; + std::vector* truePositives; + float *pNNResultsArray; + float *pcandidatesToNNClassifyIndexArray; + std::vector *candidatesToNNClassifyIndexVector; + std::vector * candidatesToNNClassifyVector; nnClassifyStruct nnClassifyStructInstance; @@ -91,25 +92,28 @@ class NNClassifier cl_context context; cl_command_queue commandQueue; cl_program program; - cl_mem inputBuffer; - cl_mem outputBuffer; - cl_kernel variance_ensemble_kernel; - cl_kernel kernel_intgegral_cols_en, kernel_intgegral_rows_en; - - cl_mem oclSrcData; - cl_mem oclbuffWindows; - - cl_mem oclbuffWindowsOffset; - cl_mem oclbufffeatureOffsets; - cl_mem oclbuffDetectionResultfeatureVectors; - cl_mem oclbuffDetectionResultPosteriors; - cl_mem oclbuffDetectionwindowFlags; - cl_mem oclbuffPosteriors; - cl_mem oclbuffDetectionResultVarious; - cl_mem oclbuffImgData; - cl_mem oclbuffII; - cl_mem oclbuffIISqure; - size_t local_work_size[1]; + + + void VectorToArray(); + /* + * class NormalizedPatch + * { + * public: + * float values[TLD_PATCH_SIZE *TLD_PATCH_SIZE]; + * bool positive; + *}; + */ + float *pSrcTruePostiveData; //float values[TLD_PATCH_SIZE *TLD_PATCH_SIZE]; + float *pSrcFalsePostiveData; + + cl_mem oclbufferSrcData; + cl_mem oclbufferWindows; + cl_mem oclbuffercandidatesToNNClassifyIndexArray; + cl_mem oclbufferpNNResultsArray; + cl_mem oclbufferSrcTruePostiveData; // device memory to SrcTruePostiveData + cl_mem oclbufferSrcFalsePostiveData; // device memory to SrcFalsePostiveData + + cl_kernel kernel_nnClassifier; From b7791222cb84811e3bc7b2c6903ba00cdafffaa9 Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Fri, 15 Sep 2017 18:43:54 +0800 Subject: [PATCH 11/14] added codes for __kernel nnClassifier there are some bugs till now, there are bugs in readbuffer from device to host. maybe the kernel nnClassifier have some bugs. I will debug it using CodeXL next step. --- .../EnsembleClassifierFilter_kernel.cl.cpp | 70 +++++++++++++------ src/libopentld/tld/DetectorCascade.cpp | 12 +++- src/libopentld/tld/NNClassifier.cpp | 69 ++++++++++++++---- src/libopentld/tld/NNClassifier.h | 2 + src/libopentld/tld/switch.h | 2 +- src/opentld/main/Main.cpp | 16 ++--- 6 files changed, 127 insertions(+), 44 deletions(-) diff --git a/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp index a173a64d..4fae1db8 100644 --- a/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp +++ b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp @@ -2,42 +2,72 @@ #pragma OPENCL EXTENSION cl_amd_printf : enable +#define TLD_PATCH_SIZE 15 - -kernel void nnClassifier( +__kernel void nnClassifier( __global uchar * img, __global int* windows, __global int* WindowIndexArray, - __global float *NNResultsArray, + __global float * NNResultsArray, const unsigned int tld_window_size, const unsigned int truePostiveSize, const unsigned int falsePositiveSize, + const unsigned int CandidatesToNNClassifySize, __global float * truePostiveData, - __global float * FalsePostiveData) + __global float * FalsePostiveData, + __global float * CandidatesToNNClassifyPatches) { - int positiveID = get_global_id(0); //positiveID is one of truePostiveSize or falsePositiveSize - if (positiveID < truePostiveSize) + int ThreadID = get_global_id(0); //ThreadID is one of truePostiveSize or falsePositiveSize + + __global float *pSrc1, *pSrc2; + int size = TLD_PATCH_SIZE * TLD_PATCH_SIZE; + int CandidatesSize = CandidatesToNNClassifySize; + __global float *conf; + + pSrc2 = CandidatesToNNClassifyPatches; //TLD_PATCH_SIZE=15 + conf = NNResultsArray + ThreadID *CandidatesSize ; + if (ThreadID < truePostiveSize && ThreadID < (truePostiveSize + truePostiveSize)) { - - //TODO - - + pSrc1 = truePostiveData + ThreadID * size; //TLD_PATCH_SIZE=15 + for (int j = 0; j < CandidatesSize; j++) + { + double corr = 0; + double norm1 = 0; + double norm2 = 0; + for (int i = 0; i < size; i++) + { + corr += pSrc1[i] * pSrc2[CandidatesSize*j + i]; + norm1 += pSrc1[i] * pSrc1[i]; + norm2 += pSrc2[j*CandidatesSize + i] * pSrc2[CandidatesSize*j + i]; + } + // normalization to <0,1> //return (corr / sqrt(norm1 * norm2) + 1) / 2.0; + *conf++ = (corr / sqrt(norm1 * norm2) + 1) / 2.0; + } } + else if(ThreadID > truePostiveSize && ThreadID < (truePostiveSize + truePostiveSize)) + { + pSrc1 = FalsePostiveData+ (ThreadID- truePostiveSize) * size; //TLD_PATCH_SIZE=15 + for (int j = 0; j < CandidatesSize; j++) + { + double corr = 0; + double norm1 = 0; + double norm2 = 0; + for (int i = 0; i < size; i++) + { + corr += pSrc1[i] * pSrc2[i]; + norm1 += pSrc1[i] * pSrc1[i]; + norm2 += pSrc2[i] * pSrc2[i]; + } + // normalization to <0,1> //return (corr / sqrt(norm1 * norm2) + 1) / 2.0; + *conf++ = (corr / sqrt(norm1 * norm2) + 1) / 2.0; + } + } // tld_window_size = 5 - int bbox0 = windows[positiveID * 5 + 0]; //x - int bbox1 = windows[positiveID * 5 + 1]; // y - int bbox2 = windows[positiveID * 5 + 2]; // width - int bbox3 = windows[positiveID * 5 + 3]; //height - int bbox4 = windows[positiveID * 5 + 4]; - float patchValue[225]; + //resize(image(source(x, y, width, height), 15 * 15, patchValue); //opencv resize function with bilinear method. - - - - } diff --git a/src/libopentld/tld/DetectorCascade.cpp b/src/libopentld/tld/DetectorCascade.cpp index 86f63b9a..65011def 100644 --- a/src/libopentld/tld/DetectorCascade.cpp +++ b/src/libopentld/tld/DetectorCascade.cpp @@ -532,9 +532,11 @@ namespace tld } } - + if (nnClassifier->candidatesToNNClassifyIndexVector->size()==0) + goto EndNNclassify; nnClassifier->clNNFilter(img); + for (int i = 0; i < nnClassifier->candidatesToNNClassifyVector->size(); i++) @@ -542,8 +544,11 @@ namespace tld if (nnClassifier->candidatesToNNClassifyVector->at(i).conf > nnClassifier->thetaTP) detectionResult->confidentIndices->push_back(nnClassifier->candidatesToNNClassifyVector->at(i).index); } - //v.clear(); - //v.shrink_to_fit(); + + + + + //·µ»Ø @@ -582,6 +587,7 @@ namespace tld //printf("confident_size %d\n\n", detectionResult->confidentIndices->size());//zhaodc //Cluster + EndNNclassify: clustering->clusterConfidentIndices(); detectionResult->containsValidData = true; diff --git a/src/libopentld/tld/NNClassifier.cpp b/src/libopentld/tld/NNClassifier.cpp index 067d4380..c10ec603 100644 --- a/src/libopentld/tld/NNClassifier.cpp +++ b/src/libopentld/tld/NNClassifier.cpp @@ -47,6 +47,7 @@ namespace tld candidatesToNNClassifyVector = new vector(); // another method is to array candidatesToNNClassifyIndexVector = new vector(); pcandidatesToNNClassifyIndexArray = NULL; + pcandidatesToNNClassifyPatches = NULL; pNNResultsArray = NULL; kernel_nnClassifier = NULL; pSrcTruePostiveData = NULL; @@ -210,21 +211,34 @@ namespace tld bool NNClassifier::clNNFilter(const cv::Mat &img) { - cl_event events[1]; + cl_event events[2]; int truePostiveSize = truePositives->size(); int falsePositiveSize = falsePositives->size(); - - if (0 == candidatesToNNClassifyIndexVector->size()) - return false; + int CandidatesToNNClassifySize = candidatesToNNClassifyIndexVector->size(); + + pcandidatesToNNClassifyPatches = new float [TLD_PATCH_SIZE * TLD_PATCH_SIZE *candidatesToNNClassifyIndexVector->size()]; + float * pPatches = pcandidatesToNNClassifyPatches; + pNNResultsArray = new float[ (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size()]; + for (int i = 0; i < (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size(); i++) + pNNResultsArray[i] = 0.0f; pcandidatesToNNClassifyIndexArray = new float[candidatesToNNClassifyIndexVector->size()]; - float *p = pcandidatesToNNClassifyIndexArray; - + float *pIndexArray = pcandidatesToNNClassifyIndexArray; + NormalizedPatch patch; for (int i = 0; i < candidatesToNNClassifyIndexVector->size(); i++) - *p++ = candidatesToNNClassifyIndexVector->at(i); + { + *pIndexArray++ = candidatesToNNClassifyIndexVector->at(i); + int *bbox = &windows[TLD_WINDOW_SIZE * i]; + tldExtractNormalizedPatchBB(img, bbox, patch.values); + float *pdest = patch.values; + for (int j = 0; j < TLD_PATCH_SIZE*TLD_PATCH_SIZE; j++) + { + *pPatches++ = *pdest++; + } + } printf("..................\n"); printf("truePositives->size()=%d \t,falsePositives->size()=%d\n", truePositives->size(), falsePositives->size()); printf("..................\n"); @@ -250,14 +264,17 @@ namespace tld for (int i = 0, count = 0; i < this->candidatesToNNClassifyVector->size(); i++) { candidatesToNNClassifyVector->at(i).conf = 1.0f; - candidatesToNNClassifyVector->at(i).index = i; + candidatesToNNClassifyVector->at(i).index = candidatesToNNClassifyIndexVector->at(i); candidatesToNNClassifyVector->at(i).flag = true; } goto EndofFuction; } - //case three + //case three + //if (truePositives->empty() && !falsePositives->empty()) + + //case four if (!truePositives->empty() && !falsePositives->empty()) { @@ -281,7 +298,10 @@ namespace tld for (int j = 0; j < TLD_PATCH_SIZE *TLD_PATCH_SIZE; j++) *p++ = *q++; } + + // End Vector to Array + oclbufferCandidatesToNNClassifyPatches = clCreateBuffer(context, CL_MEM_READ_ONLY, TLD_PATCH_SIZE * TLD_PATCH_SIZE *candidatesToNNClassifyIndexVector->size() *sizeof(int), (void*)pcandidatesToNNClassifyPatches, NULL); oclbufferWindows = clCreateBuffer(context, CL_MEM_READ_ONLY, TLD_WINDOW_SIZE * numWindows *sizeof(int), (void*) windows, NULL); oclbufferSrcData = clCreateBuffer(context, CL_MEM_READ_ONLY, (img.cols) * (img.rows) * sizeof(uchar), (void*)img.data, NULL); oclbuffercandidatesToNNClassifyIndexArray = clCreateBuffer(context, CL_MEM_READ_ONLY, candidatesToNNClassifyVector->size() * sizeof(int), (void*)pcandidatesToNNClassifyIndexArray, NULL); @@ -304,8 +324,10 @@ namespace tld status = clSetKernelArg(kernel_nnClassifier, 4, sizeof(int), (void *)&tld_window_size); status = clSetKernelArg(kernel_nnClassifier, 5, sizeof(int), (void *)&truePostiveSize); status = clSetKernelArg(kernel_nnClassifier, 6, sizeof(int), (void *)&falsePositiveSize); - status = clSetKernelArg(kernel_nnClassifier, 7, sizeof(cl_mem), (void *)&oclbufferSrcTruePostiveData); - status = clSetKernelArg(kernel_nnClassifier, 8, sizeof(cl_mem), (void *)&oclbufferSrcFalsePostiveData); + status = clSetKernelArg(kernel_nnClassifier, 7, sizeof(int), (void *)&falsePositiveSize); + status = clSetKernelArg(kernel_nnClassifier, 8, sizeof(cl_mem), (void *)&oclbufferSrcTruePostiveData); + status = clSetKernelArg(kernel_nnClassifier, 9, sizeof(cl_mem), (void *)&oclbufferSrcFalsePostiveData); + status = clSetKernelArg(kernel_nnClassifier, 10, sizeof(cl_mem), (void *)&oclbufferCandidatesToNNClassifyPatches); //status = clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *)&TLD_WINDOW_OFFSET_SIZE); @@ -328,7 +350,24 @@ namespace tld } //cout << "o" << endl; + + status = clEnqueueReadBuffer(commandQueue, oclbufferpNNResultsArray, CL_FALSE, 0, (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size()* sizeof(float), (void *)pNNResultsArray, 0, NULL, &events[0]); + status = clWaitForEvents(1, &events[0]); + if (status != CL_SUCCESS) + { + cout << "Error:NNclassify EnqueueNDRangeKernel!" << endl; + //return false; + } + + printf("Reading data from GPU *************************\n"); + for (int i = 0; i <(truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size(); i++) + + printf("detectionResult[%d] is %d\n", i, pNNResultsArray[i]); + + + status = clReleaseEvent(events[0]); + status = clReleaseEvent(events[1]); //printf("end using GPU*************************\n"); //for (int i = 0; i < numWindows; i++) @@ -343,7 +382,12 @@ namespace tld pSrcTruePostiveData = NULL; delete pSrcFalsePostiveData; pSrcFalsePostiveData = NULL; - + delete pNNResultsArray; + pNNResultsArray = NULL; + delete pcandidatesToNNClassifyIndexArray; + pcandidatesToNNClassifyIndexArray = NULL; + delete pcandidatesToNNClassifyPatches; + pcandidatesToNNClassifyPatches = NULL; clReleaseKernel(kernel_nnClassifier); @@ -352,6 +396,7 @@ namespace tld clReleaseMemObject(oclbufferWindows); clReleaseMemObject(oclbuffercandidatesToNNClassifyIndexArray); clReleaseMemObject(oclbufferpNNResultsArray); + clReleaseMemObject(oclbufferCandidatesToNNClassifyPatches); diff --git a/src/libopentld/tld/NNClassifier.h b/src/libopentld/tld/NNClassifier.h index 7ffacf8d..e3da4078 100644 --- a/src/libopentld/tld/NNClassifier.h +++ b/src/libopentld/tld/NNClassifier.h @@ -65,6 +65,7 @@ class NNClassifier std::vector* truePositives; float *pNNResultsArray; float *pcandidatesToNNClassifyIndexArray; + float *pcandidatesToNNClassifyPatches; std::vector *candidatesToNNClassifyIndexVector; std::vector * candidatesToNNClassifyVector; @@ -112,6 +113,7 @@ class NNClassifier cl_mem oclbufferpNNResultsArray; cl_mem oclbufferSrcTruePostiveData; // device memory to SrcTruePostiveData cl_mem oclbufferSrcFalsePostiveData; // device memory to SrcFalsePostiveData + cl_mem oclbufferCandidatesToNNClassifyPatches; // candidatest to NNClassify patches from host to devices. cl_kernel kernel_nnClassifier; diff --git a/src/libopentld/tld/switch.h b/src/libopentld/tld/switch.h index 9ed8667a..77b87890 100644 --- a/src/libopentld/tld/switch.h +++ b/src/libopentld/tld/switch.h @@ -5,7 +5,7 @@ #define clIntegral_m 1 #define clIntegral_m_1 0 #define clOverlap 0 -#define clNNClassifier 0 +#define clNNClassifier 1 #define PrintTime_overlap 0 #define PrintTime_detect 0 diff --git a/src/opentld/main/Main.cpp b/src/opentld/main/Main.cpp index bba5dcd1..110e3aab 100644 --- a/src/opentld/main/Main.cpp +++ b/src/opentld/main/Main.cpp @@ -66,14 +66,14 @@ void Main::doWork() initialBB = new int[4]; } - initialBB[0] = box.x; - initialBB[1] = box.y; - initialBB[2] = box.width; - initialBB[3] = box.height; - //initialBB[0] = 650; - //initialBB[1] = 550; - //initialBB[2] = 300; - //initialBB[3] = 300; + //initialBB[0] = box.x; + //initialBB[1] = box.y; + //initialBB[2] = box.width; + //initialBB[3] = box.height; + initialBB[0] = 650; + initialBB[1] = 550; + initialBB[2] = 300; + initialBB[3] = 300; //initialBB[0] = 120; //initialBB[1] = 120; //initialBB[2] = 110; From 824af2cbbb00b87861235d11e302e2bbff621be0 Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Mon, 18 Sep 2017 17:09:16 +0800 Subject: [PATCH 12/14] Finished NNclassifer using OpenCL finished NNclassifer using OpenCL. --- .../EnsembleClassifierFilter_kernel.cl.cpp | 38 +++++--- src/libopentld/tld/NNClassifier.cpp | 94 +++++++++++++------ src/opentld/main/Main.cpp | 16 ++-- 3 files changed, 97 insertions(+), 51 deletions(-) diff --git a/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp index 4fae1db8..172e0d50 100644 --- a/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp +++ b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp @@ -6,11 +6,11 @@ __kernel void nnClassifier( - __global uchar * img, - __global int* windows, - __global int* WindowIndexArray, + //__global uchar * img, + //__global int* windows, + //__global int* WindowIndexArray, __global float * NNResultsArray, - const unsigned int tld_window_size, + //const unsigned int tld_window_size, const unsigned int truePostiveSize, const unsigned int falsePositiveSize, const unsigned int CandidatesToNNClassifySize, @@ -22,46 +22,60 @@ __kernel void nnClassifier( int ThreadID = get_global_id(0); //ThreadID is one of truePostiveSize or falsePositiveSize __global float *pSrc1, *pSrc2; - int size = TLD_PATCH_SIZE * TLD_PATCH_SIZE; + int patchSize = TLD_PATCH_SIZE * TLD_PATCH_SIZE; int CandidatesSize = CandidatesToNNClassifySize; __global float *conf; pSrc2 = CandidatesToNNClassifyPatches; //TLD_PATCH_SIZE=15 conf = NNResultsArray + ThreadID *CandidatesSize ; - if (ThreadID < truePostiveSize && ThreadID < (truePostiveSize + truePostiveSize)) + if (ThreadID < truePostiveSize && ThreadID < (truePostiveSize + falsePositiveSize)) { - pSrc1 = truePostiveData + ThreadID * size; //TLD_PATCH_SIZE=15 + pSrc1 = truePostiveData + ThreadID * patchSize; //TLD_PATCH_SIZE=15 for (int j = 0; j < CandidatesSize; j++) { double corr = 0; double norm1 = 0; double norm2 = 0; - for (int i = 0; i < size; i++) + for (int i = 0; i < patchSize; i++) { corr += pSrc1[i] * pSrc2[CandidatesSize*j + i]; norm1 += pSrc1[i] * pSrc1[i]; norm2 += pSrc2[j*CandidatesSize + i] * pSrc2[CandidatesSize*j + i]; } // normalization to <0,1> //return (corr / sqrt(norm1 * norm2) + 1) / 2.0; - *conf++ = (corr / sqrt(norm1 * norm2) + 1) / 2.0; + + if (norm1 == 0.0 || norm2 == 0.0) + { + norm1 = 1.0; + norm2 = 1.0; + } + else + *conf++ = (corr / sqrt(norm1 * norm2) + 1) / 2.0; } } - else if(ThreadID > truePostiveSize && ThreadID < (truePostiveSize + truePostiveSize)) + else if(ThreadID > truePostiveSize && ThreadID < (truePostiveSize + falsePositiveSize)) { - pSrc1 = FalsePostiveData+ (ThreadID- truePostiveSize) * size; //TLD_PATCH_SIZE=15 + pSrc1 = FalsePostiveData+ (ThreadID- truePostiveSize) * patchSize; //TLD_PATCH_SIZE=15 for (int j = 0; j < CandidatesSize; j++) { double corr = 0; double norm1 = 0; double norm2 = 0; - for (int i = 0; i < size; i++) + for (int i = 0; i < patchSize; i++) { corr += pSrc1[i] * pSrc2[i]; norm1 += pSrc1[i] * pSrc1[i]; norm2 += pSrc2[i] * pSrc2[i]; } // normalization to <0,1> //return (corr / sqrt(norm1 * norm2) + 1) / 2.0; + if(norm1==0.0 || norm2 ==0.0) + { + norm1 = 1.0; + norm2 = 1.0; + } + else *conf++ = (corr / sqrt(norm1 * norm2) + 1) / 2.0; + } } // tld_window_size = 5 diff --git a/src/libopentld/tld/NNClassifier.cpp b/src/libopentld/tld/NNClassifier.cpp index c10ec603..c7b87341 100644 --- a/src/libopentld/tld/NNClassifier.cpp +++ b/src/libopentld/tld/NNClassifier.cpp @@ -219,7 +219,8 @@ namespace tld pcandidatesToNNClassifyPatches = new float [TLD_PATCH_SIZE * TLD_PATCH_SIZE *candidatesToNNClassifyIndexVector->size()]; float * pPatches = pcandidatesToNNClassifyPatches; - + float maxPositiveValue ; + float maxFalseValue; pNNResultsArray = new float[ (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size()]; for (int i = 0; i < (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size(); i++) pNNResultsArray[i] = 0.0f; @@ -236,7 +237,6 @@ namespace tld for (int j = 0; j < TLD_PATCH_SIZE*TLD_PATCH_SIZE; j++) { *pPatches++ = *pdest++; - } } printf("..................\n"); @@ -292,7 +292,7 @@ namespace tld } pSrcFalsePostiveData = new float[falsePositiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE]; p = pSrcFalsePostiveData; - for (int i = 0; i < truePostiveSize; i++) + for (int i = 0; i < falsePositiveSize; i++) { q = falsePositives->at(i).values; for (int j = 0; j < TLD_PATCH_SIZE *TLD_PATCH_SIZE; j++) @@ -301,14 +301,13 @@ namespace tld // End Vector to Array - oclbufferCandidatesToNNClassifyPatches = clCreateBuffer(context, CL_MEM_READ_ONLY, TLD_PATCH_SIZE * TLD_PATCH_SIZE *candidatesToNNClassifyIndexVector->size() *sizeof(int), (void*)pcandidatesToNNClassifyPatches, NULL); - oclbufferWindows = clCreateBuffer(context, CL_MEM_READ_ONLY, TLD_WINDOW_SIZE * numWindows *sizeof(int), (void*) windows, NULL); - oclbufferSrcData = clCreateBuffer(context, CL_MEM_READ_ONLY, (img.cols) * (img.rows) * sizeof(uchar), (void*)img.data, NULL); - oclbuffercandidatesToNNClassifyIndexArray = clCreateBuffer(context, CL_MEM_READ_ONLY, candidatesToNNClassifyVector->size() * sizeof(int), (void*)pcandidatesToNNClassifyIndexArray, NULL); - oclbufferpNNResultsArray = clCreateBuffer(context, CL_MEM_READ_ONLY, (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size()* sizeof(float), (void*)pNNResultsArray, NULL); - - oclbufferSrcTruePostiveData = clCreateBuffer(context, CL_MEM_READ_ONLY, truePostiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE* sizeof(float), (void*)pSrcTruePostiveData, NULL); - oclbufferSrcFalsePostiveData = clCreateBuffer(context, CL_MEM_READ_ONLY, falsePositiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE* sizeof(float), (void*)pSrcFalsePostiveData, NULL); + // oclbufferWindows = clCreateBuffer(context, CL_MEM_READ_ONLY, TLD_WINDOW_SIZE * numWindows *sizeof(int), (void*) windows, NULL); + // oclbufferSrcData = clCreateBuffer(context, CL_MEM_READ_ONLY, (img.cols) * (img.rows) * sizeof(uchar), (void*)img.data, NULL); + // oclbuffercandidatesToNNClassifyIndexArray = clCreateBuffer(context, CL_MEM_READ_ONLY, candidatesToNNClassifyVector->size() * sizeof(int), (void*)pcandidatesToNNClassifyIndexArray, NULL); + oclbufferpNNResultsArray = clCreateBuffer(context, CL_MEM_READ_WRITE |CL_MEM_USE_HOST_PTR, (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size()* sizeof(float), (void*)pNNResultsArray, NULL); + oclbufferSrcTruePostiveData = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, truePostiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE* sizeof(float), (void*)pSrcTruePostiveData, NULL); + oclbufferSrcFalsePostiveData = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, falsePositiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE* sizeof(float), (void*)pSrcFalsePostiveData, NULL); + oclbufferCandidatesToNNClassifyPatches = clCreateBuffer(context, CL_MEM_READ_WRITE| CL_MEM_USE_HOST_PTR, TLD_PATCH_SIZE * TLD_PATCH_SIZE *candidatesToNNClassifyIndexVector->size() *sizeof(int), (void*)pcandidatesToNNClassifyPatches, NULL); @@ -317,17 +316,17 @@ namespace tld // /*Step 8: Create kernel object */ /*Step 9: Sets Kernel arguments.*/ - status = clSetKernelArg(kernel_nnClassifier, 0, sizeof(cl_mem), (void *)&oclbufferSrcData); - status = clSetKernelArg(kernel_nnClassifier, 1, sizeof(cl_mem), (void *)&oclbufferWindows); - status = clSetKernelArg(kernel_nnClassifier, 2, sizeof(cl_mem), (void *)&oclbuffercandidatesToNNClassifyIndexArray); - status = clSetKernelArg(kernel_nnClassifier, 3, sizeof(cl_mem), (void *)&oclbufferpNNResultsArray); - status = clSetKernelArg(kernel_nnClassifier, 4, sizeof(int), (void *)&tld_window_size); - status = clSetKernelArg(kernel_nnClassifier, 5, sizeof(int), (void *)&truePostiveSize); - status = clSetKernelArg(kernel_nnClassifier, 6, sizeof(int), (void *)&falsePositiveSize); - status = clSetKernelArg(kernel_nnClassifier, 7, sizeof(int), (void *)&falsePositiveSize); - status = clSetKernelArg(kernel_nnClassifier, 8, sizeof(cl_mem), (void *)&oclbufferSrcTruePostiveData); - status = clSetKernelArg(kernel_nnClassifier, 9, sizeof(cl_mem), (void *)&oclbufferSrcFalsePostiveData); - status = clSetKernelArg(kernel_nnClassifier, 10, sizeof(cl_mem), (void *)&oclbufferCandidatesToNNClassifyPatches); + //status = clSetKernelArg(kernel_nnClassifier, 0, sizeof(cl_mem), (void *)&oclbufferSrcData); + //status = clSetKernelArg(kernel_nnClassifier, 1, sizeof(cl_mem), (void *)&oclbufferWindows); + //status = clSetKernelArg(kernel_nnClassifier, 2, sizeof(cl_mem), (void *)&oclbuffercandidatesToNNClassifyIndexArray); + status = clSetKernelArg(kernel_nnClassifier, 0, sizeof(cl_mem), (void *)&oclbufferpNNResultsArray); + //status = clSetKernelArg(kernel_nnClassifier, 4, sizeof(int), (void *)&tld_window_size); + status = clSetKernelArg(kernel_nnClassifier, 1, sizeof(int), (void *)&truePostiveSize); + status = clSetKernelArg(kernel_nnClassifier, 2, sizeof(int), (void *)&falsePositiveSize); + status = clSetKernelArg(kernel_nnClassifier, 3, sizeof(int), (void *)&CandidatesToNNClassifySize); + status = clSetKernelArg(kernel_nnClassifier, 4, sizeof(cl_mem), (void *)&oclbufferSrcTruePostiveData); + status = clSetKernelArg(kernel_nnClassifier, 5, sizeof(cl_mem), (void *)&oclbufferSrcFalsePostiveData); + status = clSetKernelArg(kernel_nnClassifier, 6, sizeof(cl_mem), (void *)&oclbufferCandidatesToNNClassifyPatches); //status = clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *)&TLD_WINDOW_OFFSET_SIZE); @@ -360,14 +359,47 @@ namespace tld } printf("Reading data from GPU *************************\n"); - for (int i = 0; i <(truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size(); i++) + for (int i = 0; i < (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size(); i++) + { + printf("candidatesToNNClassifyIndexVector->size()=%d£¬ detectionResult[%d] is %f\n", candidatesToNNClassifyIndexVector->size(),i, pNNResultsArray[i]); + + + } - printf("detectionResult[%d] is %d\n", i, pNNResultsArray[i]); + for (int i = 0; i < candidatesToNNClassifyIndexVector->size();i++) + { + maxPositiveValue = pNNResultsArray[i]; + maxFalseValue = pNNResultsArray[i+ truePostiveSize*candidatesToNNClassifyIndexVector->size()]; + for (int j = 1; j < truePostiveSize; j++) + { + if ( maxPositiveValue < pNNResultsArray[j*candidatesToNNClassifyIndexVector->size()]) + maxPositiveValue = pNNResultsArray[j*candidatesToNNClassifyIndexVector->size()]; + + } + for (int j = truePostiveSize; j < truePostiveSize+falsePositiveSize; j++) + { + if (maxFalseValue < pNNResultsArray[j*candidatesToNNClassifyIndexVector->size()]) + maxFalseValue = pNNResultsArray[j*candidatesToNNClassifyIndexVector->size()]; + + } + float dN = 1 - maxFalseValue; + float dP = 1 - maxPositiveValue; + + float conf = dN / (dN + dP); + // if (conf > thetaTP) + //{ + // return false; + //} + candidatesToNNClassifyVector->at(i).conf = conf; + candidatesToNNClassifyVector->at(i).index = candidatesToNNClassifyIndexVector->at(i); + candidatesToNNClassifyVector->at(i).flag = true; + + } - status = clReleaseEvent(events[0]); - status = clReleaseEvent(events[1]); + //status = clReleaseEvent(events[0]); + //status = clReleaseEvent(events[1]); //printf("end using GPU*************************\n"); //for (int i = 0; i < numWindows; i++) @@ -390,11 +422,11 @@ namespace tld pcandidatesToNNClassifyPatches = NULL; - clReleaseKernel(kernel_nnClassifier); - clReleaseProgram(program); - clReleaseMemObject(oclbufferSrcData); - clReleaseMemObject(oclbufferWindows); - clReleaseMemObject(oclbuffercandidatesToNNClassifyIndexArray); + // clReleaseKernel(kernel_nnClassifier); + // clReleaseProgram(program); + // clReleaseMemObject(oclbufferSrcData); + // clReleaseMemObject(oclbufferWindows); + // clReleaseMemObject(oclbuffercandidatesToNNClassifyIndexArray); clReleaseMemObject(oclbufferpNNResultsArray); clReleaseMemObject(oclbufferCandidatesToNNClassifyPatches); diff --git a/src/opentld/main/Main.cpp b/src/opentld/main/Main.cpp index 110e3aab..bba5dcd1 100644 --- a/src/opentld/main/Main.cpp +++ b/src/opentld/main/Main.cpp @@ -66,14 +66,14 @@ void Main::doWork() initialBB = new int[4]; } - //initialBB[0] = box.x; - //initialBB[1] = box.y; - //initialBB[2] = box.width; - //initialBB[3] = box.height; - initialBB[0] = 650; - initialBB[1] = 550; - initialBB[2] = 300; - initialBB[3] = 300; + initialBB[0] = box.x; + initialBB[1] = box.y; + initialBB[2] = box.width; + initialBB[3] = box.height; + //initialBB[0] = 650; + //initialBB[1] = 550; + //initialBB[2] = 300; + //initialBB[3] = 300; //initialBB[0] = 120; //initialBB[1] = 120; //initialBB[2] = 110; From e098062c59ee6a63805141459f0f52647258fa79 Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Tue, 19 Sep 2017 13:55:33 +0800 Subject: [PATCH 13/14] Modified Switch.h file to switch CPU and GPU 1.fixed bugs in varaiceFilter.cpp in CPU model. 2. fixed one bugs in kernel named cltldOverlapRect_kernel.cpp in using goto statement. must write one semicolon. 3. delete some statements without usage. or delete unused codes. --- src/libopentld/CMakeLists.txt | 3 +- .../EnsembleClassifierFilter_kernel.cl.cpp | 4 -- .../kernel/cltldOverlapRect_kernel.cpp | 9 ++-- src/libopentld/tld/DetectorCascade.cpp | 6 ++- src/libopentld/tld/NNClassifier.cpp | 49 +++++++------------ src/libopentld/tld/TLD.cpp | 18 ++++++- src/libopentld/tld/VarianceFilter.cpp | 7 +-- src/libopentld/tld/switch.h | 15 ++++-- 8 files changed, 58 insertions(+), 53 deletions(-) diff --git a/src/libopentld/CMakeLists.txt b/src/libopentld/CMakeLists.txt index b9aa139f..193c9297 100644 --- a/src/libopentld/CMakeLists.txt +++ b/src/libopentld/CMakeLists.txt @@ -42,7 +42,8 @@ add_library(libopentld tld/NormalizedPatch.h tld/TLD.h tld/TLDUtil.h - tld/VarianceFilter.h) + tld/VarianceFilter.h + tld/switch.h) target_link_libraries(libopentld ${OpenCV_LIBS} ${OPENCL_LIBRARIES}) diff --git a/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp index 172e0d50..6a8b6a15 100644 --- a/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp +++ b/src/libopentld/kernel/EnsembleClassifierFilter_kernel.cl.cpp @@ -6,11 +6,7 @@ __kernel void nnClassifier( - //__global uchar * img, - //__global int* windows, - //__global int* WindowIndexArray, __global float * NNResultsArray, - //const unsigned int tld_window_size, const unsigned int truePostiveSize, const unsigned int falsePositiveSize, const unsigned int CandidatesToNNClassifySize, diff --git a/src/libopentld/kernel/cltldOverlapRect_kernel.cpp b/src/libopentld/kernel/cltldOverlapRect_kernel.cpp index 7fd3975d..55486fb9 100644 --- a/src/libopentld/kernel/cltldOverlapRect_kernel.cpp +++ b/src/libopentld/kernel/cltldOverlapRect_kernel.cpp @@ -8,12 +8,10 @@ __kernel void cltldOverlapRect( const unsigned int bb11, const unsigned int bb12, const unsigned int bb13, - __global int* oclbuffWindowsOffset, - __global float* overlap , - + __global float* overlap, const unsigned int TLD_WINDOW_SIZE - ) + ) { int windowIdx = get_global_id(0); int min,max,sum1,sum2; @@ -62,7 +60,8 @@ __kernel void cltldOverlapRect( overlap[windowIdx] = intersection / (float)(area1 + area2 - intersection); } - finished: + finished: + ; //if(overlap[windowIdx]!=0.0) //if( windowIdx==249404 ) //{ diff --git a/src/libopentld/tld/DetectorCascade.cpp b/src/libopentld/tld/DetectorCascade.cpp index 65011def..de2f62b5 100644 --- a/src/libopentld/tld/DetectorCascade.cpp +++ b/src/libopentld/tld/DetectorCascade.cpp @@ -414,7 +414,7 @@ namespace tld free(program_log); - cout << "DetectorCascade::oclBuildKernelError: Getting platforms!" << endl; + cout << "DetectorCascade::oclBuildKernelError:clBuildProgram !" << endl; return false; } @@ -534,8 +534,10 @@ namespace tld } if (nnClassifier->candidatesToNNClassifyIndexVector->size()==0) goto EndNNclassify; - nnClassifier->clNNFilter(img); + + nnClassifier->clNNFilter(img); + diff --git a/src/libopentld/tld/NNClassifier.cpp b/src/libopentld/tld/NNClassifier.cpp index c7b87341..7340a199 100644 --- a/src/libopentld/tld/NNClassifier.cpp +++ b/src/libopentld/tld/NNClassifier.cpp @@ -136,9 +136,9 @@ namespace tld { return 1; } - printf("..................\n"); - printf("truePositives->size()=%d \t,falsePositives->size()=%d\n", truePositives->size(), falsePositives->size()); - printf("..................\n"); + //printf("..................\n"); + //printf("truePositives->size()=%d \t,falsePositives->size()=%d\n", truePositives->size(), falsePositives->size()); + //printf("..................\n"); float ccorr_max_p = 0; //Compare patch to positive patches @@ -239,9 +239,9 @@ namespace tld *pPatches++ = *pdest++; } } - printf("..................\n"); - printf("truePositives->size()=%d \t,falsePositives->size()=%d\n", truePositives->size(), falsePositives->size()); - printf("..................\n"); + //printf("..................\n"); + //printf("truePositives->size()=%d \t,falsePositives->size()=%d\n", truePositives->size(), falsePositives->size()); + //printf("..................\n"); // case one if (truePositives->empty()) @@ -251,8 +251,8 @@ namespace tld for (int i = 0, count = 0; i < this->candidatesToNNClassifyVector->size(); i++) { candidatesToNNClassifyVector->at(i).conf = 0.0f; - candidatesToNNClassifyVector->at(i).index = i; - candidatesToNNClassifyVector->at(i).flag = false; + //candidatesToNNClassifyVector->at(i).index = i; + //candidatesToNNClassifyVector->at(i).flag = false; } goto EndofFuction; @@ -264,14 +264,14 @@ namespace tld for (int i = 0, count = 0; i < this->candidatesToNNClassifyVector->size(); i++) { candidatesToNNClassifyVector->at(i).conf = 1.0f; - candidatesToNNClassifyVector->at(i).index = candidatesToNNClassifyIndexVector->at(i); - candidatesToNNClassifyVector->at(i).flag = true; + //candidatesToNNClassifyVector->at(i).index = candidatesToNNClassifyIndexVector->at(i); + //candidatesToNNClassifyVector->at(i).flag = true; } goto EndofFuction; } - //case three + //case three TODO£º //if (truePositives->empty() && !falsePositives->empty()) //case four @@ -298,29 +298,16 @@ namespace tld for (int j = 0; j < TLD_PATCH_SIZE *TLD_PATCH_SIZE; j++) *p++ = *q++; } - - + // End Vector to Array - // oclbufferWindows = clCreateBuffer(context, CL_MEM_READ_ONLY, TLD_WINDOW_SIZE * numWindows *sizeof(int), (void*) windows, NULL); - // oclbufferSrcData = clCreateBuffer(context, CL_MEM_READ_ONLY, (img.cols) * (img.rows) * sizeof(uchar), (void*)img.data, NULL); - // oclbuffercandidatesToNNClassifyIndexArray = clCreateBuffer(context, CL_MEM_READ_ONLY, candidatesToNNClassifyVector->size() * sizeof(int), (void*)pcandidatesToNNClassifyIndexArray, NULL); - oclbufferpNNResultsArray = clCreateBuffer(context, CL_MEM_READ_WRITE |CL_MEM_USE_HOST_PTR, (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size()* sizeof(float), (void*)pNNResultsArray, NULL); + oclbufferpNNResultsArray = clCreateBuffer(context, CL_MEM_READ_WRITE |CL_MEM_USE_HOST_PTR, (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size()* sizeof(float), (void*)pNNResultsArray, NULL); oclbufferSrcTruePostiveData = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, truePostiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE* sizeof(float), (void*)pSrcTruePostiveData, NULL); oclbufferSrcFalsePostiveData = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, falsePositiveSize * TLD_PATCH_SIZE *TLD_PATCH_SIZE* sizeof(float), (void*)pSrcFalsePostiveData, NULL); oclbufferCandidatesToNNClassifyPatches = clCreateBuffer(context, CL_MEM_READ_WRITE| CL_MEM_USE_HOST_PTR, TLD_PATCH_SIZE * TLD_PATCH_SIZE *candidatesToNNClassifyIndexVector->size() *sizeof(int), (void*)pcandidatesToNNClassifyPatches, NULL); - - - - - - // + /*Step 8: Create kernel object */ /*Step 9: Sets Kernel arguments.*/ - //status = clSetKernelArg(kernel_nnClassifier, 0, sizeof(cl_mem), (void *)&oclbufferSrcData); - //status = clSetKernelArg(kernel_nnClassifier, 1, sizeof(cl_mem), (void *)&oclbufferWindows); - //status = clSetKernelArg(kernel_nnClassifier, 2, sizeof(cl_mem), (void *)&oclbuffercandidatesToNNClassifyIndexArray); - status = clSetKernelArg(kernel_nnClassifier, 0, sizeof(cl_mem), (void *)&oclbufferpNNResultsArray); - //status = clSetKernelArg(kernel_nnClassifier, 4, sizeof(int), (void *)&tld_window_size); + status = clSetKernelArg(kernel_nnClassifier, 0, sizeof(cl_mem), (void *)&oclbufferpNNResultsArray); status = clSetKernelArg(kernel_nnClassifier, 1, sizeof(int), (void *)&truePostiveSize); status = clSetKernelArg(kernel_nnClassifier, 2, sizeof(int), (void *)&falsePositiveSize); status = clSetKernelArg(kernel_nnClassifier, 3, sizeof(int), (void *)&CandidatesToNNClassifySize); @@ -331,7 +318,7 @@ namespace tld //status = clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *)&TLD_WINDOW_OFFSET_SIZE); /*Step 10: Running the kernel.*/ - printf("begore opencl kernel numWindows=%d\n", numWindows); + //printf("begore opencl kernel numWindows=%d\n", numWindows); size_t global_work_size[1] = { truePostiveSize+falsePositiveSize }; size_t local_work_size[1] = { 256 }; status = clEnqueueNDRangeKernel(commandQueue, kernel_nnClassifier, 1, NULL, global_work_size, NULL, 0, NULL, &events[0]); @@ -358,14 +345,14 @@ namespace tld //return false; } - printf("Reading data from GPU *************************\n"); + /* printf("Reading data from GPU *************************\n"); for (int i = 0; i < (truePostiveSize + falsePositiveSize)*candidatesToNNClassifyIndexVector->size(); i++) { printf("candidatesToNNClassifyIndexVector->size()=%d£¬ detectionResult[%d] is %f\n", candidatesToNNClassifyIndexVector->size(),i, pNNResultsArray[i]); } - + */ for (int i = 0; i < candidatesToNNClassifyIndexVector->size();i++) { maxPositiveValue = pNNResultsArray[i]; diff --git a/src/libopentld/tld/TLD.cpp b/src/libopentld/tld/TLD.cpp index 4343a358..a20ca60d 100644 --- a/src/libopentld/tld/TLD.cpp +++ b/src/libopentld/tld/TLD.cpp @@ -71,17 +71,31 @@ namespace tld medianFlowTracker = new MedianFlowTracker(); int tld_window_size = TLD_WINDOW_SIZE; - char *kernelName = "..\\..\\..\\src\\libopentld\\kernel\\cltldOverlapRect.cl"; + char *kernelName = "..\\..\\..\\src\\libopentld\\kernel\\cltldOverlapRect_kernel.cpp"; string sourceStr; status = convertToString2(kernelName, sourceStr); + const char *source = sourceStr.c_str(); size_t sourceSize[] = { strlen(source) }; program = clCreateProgramWithSource(context, 1, &source, sourceSize, NULL); /*Step 6: Build program. */ status = clBuildProgram(program, 1, &devices[SELECTED_DEVICE_ID], NULL, NULL, NULL); + if (status != CL_SUCCESS) + { + size_t log_size; + char* program_log; + clGetProgramBuildInfo(program, devices[SELECTED_DEVICE_ID], CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size); + program_log = (char*)malloc(log_size + 1); + clGetProgramBuildInfo(program, devices[SELECTED_DEVICE_ID], CL_PROGRAM_BUILD_LOG, log_size + 1, program_log, NULL); + printf("%s\n", program_log); + free(program_log); + + cout << "DetectorCascade::oclBuildKernelError:clBuildProgram !" << endl; + //return false; + } } TLD::~TLD() @@ -380,7 +394,7 @@ namespace tld status = clEnqueueNDRangeKernel(commandQueue, kernel, 1, NULL, global_work_size, local_work_size, 0, NULL, &events[0]); if (status != CL_SUCCESS) { - cout << "Error: EnqueueNDRangeKernel!" << endl; + cout << "Error:cltldOverlapRect_self EnqueueNDRangeKernel!" << endl; //return false; } status = clWaitForEvents(1, &events[0]); diff --git a/src/libopentld/tld/VarianceFilter.cpp b/src/libopentld/tld/VarianceFilter.cpp index 3cb0e716..0678825a 100644 --- a/src/libopentld/tld/VarianceFilter.cpp +++ b/src/libopentld/tld/VarianceFilter.cpp @@ -357,9 +357,10 @@ namespace tld if (!enabled) return; release(); - - // integralImg = new IntegralImage(img.size()); - //integralImg_squared = new IntegralImage(img.size()); +#if CPU + integralImg = new IntegralImage(img.size()); + integralImg_squared = new IntegralImage(img.size()); +#endif #if PrintTime_integral double tic = cvGetTickCount(); #endif diff --git a/src/libopentld/tld/switch.h b/src/libopentld/tld/switch.h index 77b87890..bd49f27e 100644 --- a/src/libopentld/tld/switch.h +++ b/src/libopentld/tld/switch.h @@ -1,11 +1,16 @@ -#define opencl 1 +#define opencl 0 +#if opencl +#define CPU 0 +#else +#define CPU 1 +#endif -#define clDetect 1 -#define clIntegral 1 -#define clIntegral_m 1 +#define clDetect 0 +#define clIntegral 0 +#define clIntegral_m 0 #define clIntegral_m_1 0 #define clOverlap 0 -#define clNNClassifier 1 +#define clNNClassifier 0 #define PrintTime_overlap 0 #define PrintTime_detect 0 From dae0da68bcb48da53721e1a7e8460fabf4a44430 Mon Sep 17 00:00:00 2001 From: xqch1983 Date: Tue, 19 Sep 2017 15:55:44 +0800 Subject: [PATCH 14/14] learn some operation on Github desktop learn some operation on Github desktop --- src/libopentld/tld/DetectorCascade.cpp | 4 ++-- src/libopentld/tld/TLD.cpp | 2 +- src/libopentld/tld/VarianceFilter.cpp | 18 +++++++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libopentld/tld/DetectorCascade.cpp b/src/libopentld/tld/DetectorCascade.cpp index de2f62b5..9b6564c5 100644 --- a/src/libopentld/tld/DetectorCascade.cpp +++ b/src/libopentld/tld/DetectorCascade.cpp @@ -567,8 +567,8 @@ namespace tld if (detectionResult->posteriors[i] >= 0.5f) { // printf("windowIDx= %d,count=%d\n", i, count++); - count++; - printf("Framenum = %d, I am victor %d,count=%d\n", FramNum, i, count); + //count++; + //printf("Framenum = %d, I am victor %d,count=%d\n", FramNum, i, count); if (!nnClassifier->filter(img, i)) { continue; diff --git a/src/libopentld/tld/TLD.cpp b/src/libopentld/tld/TLD.cpp index a20ca60d..8b9d788f 100644 --- a/src/libopentld/tld/TLD.cpp +++ b/src/libopentld/tld/TLD.cpp @@ -554,7 +554,7 @@ namespace tld } } } - cout << "xxxxnegativeIndices.size()xxx" << negativeIndices.size() << endl; + //cout << "xxxxnegativeIndices.size()xxx" << negativeIndices.size() << endl; // FILE *fp = fopen("C:\\Users\\kevin\\Desktop\\compResult\\opentld_opencl_overlap.txt", "a+"); // for (int i = 0; i < negativeIndices.size();i++) //fprintf(fp, "windowIdx=%d,overlap=%f\n", i, overlap[i]); diff --git a/src/libopentld/tld/VarianceFilter.cpp b/src/libopentld/tld/VarianceFilter.cpp index 0678825a..0bbcbef4 100644 --- a/src/libopentld/tld/VarianceFilter.cpp +++ b/src/libopentld/tld/VarianceFilter.cpp @@ -364,15 +364,15 @@ namespace tld #if PrintTime_integral double tic = cvGetTickCount(); #endif -#if clIntegral_m_1 - //integralImg->oclcalcIntImg(img); - //integralImg_squared->oclcalcIntImgSquare(img, true); - - oclMat temp; - temp = img; - jifentu2(temp, sum, sqsum); //(m+1,n+1)-->(m,n) - -#elif clIntegral_m +//#if clIntegral_m_1 +// //integralImg->oclcalcIntImg(img); +// //integralImg_squared->oclcalcIntImgSquare(img, true); +// +// oclMat temp; +// temp = img; +// jifentu2(temp, sum, sqsum); //(m+1,n+1)-->(m,n) + +#if clIntegral_m //integralImag_opencv( img); // (m,n)-->(m,n) integralImag_extract(img, iSumMat, fSqreSumMat); //integralImag_extract(img);