训练正常，测试异常，什么鬼？（cudnn）

前天残差中加不进去bn层

昨天残差中加不进去两个bn层

昨夜，残差中可以用两个bn层，而且训练正常，一测试，就闪退，什么鬼？

测试函数如下：

int last[10000];
float test( LeNet &net, cudaStream_t stream, cifar10DatasetTEST &Dataset) {//使用10000
float *d_inputs;
unsigned *d_labels;

for (int x = 0; x<10000; x++)
{
last[x] = 0;
}
//输出的预测概率，softmax后的结果，损失，预测概率的梯度（作为LeNet反向传播的输入）
float *d_logits, *d_prob, *loss, *dlogits;
int num_classes = 10;
cudaMalloc(&d_inputs, 5 * 32 * 32 * sizeof(float));
/*cudaMalloc(&d_inputs, 3* 32*32 * sizeof(float));*/
cudaMalloc(&d_labels, 1*sizeof(unsigned));
cudaMalloc(&d_logits, 1*10*sizeof(float));
cudaMalloc(&d_prob, 1*10*sizeof(float));
cudaMalloc(&loss, 1*sizeof(float));
cudaMalloc(&dlogits, 1*10*sizeof(float));

std::vector<float> h_loss(1); std::vector<float> h_prob(10);

for (int epoch = 0; epoch<10000; epoch++) {
auto batch = Dataset.next_batch(1);
std::vector<unsigned> h_labels(1);//取了一个图像一个标签，也可以是train数据取10000

for (int batch_idx = 0; batch_idx<1; batch_idx++) {
cudaMemcpy(d_inputs + batch_idx * 32 * 32 * 5, batch[batch_idx].image.data(), 32 * 32 * 5 * sizeof(float), cudaMemcpyHostToDevice);
/*cudaMemcpy(d_inputs + batch_idx * 32 * 32 * 3, batch[batch_idx].image.data(), 32 * 32 * 3 * sizeof(float), cudaMemcpyHostToDevice);*/
h_labels[batch_idx] = batch[batch_idx].label;
}
cudaMemcpy(d_labels, h_labels.data(),
1*sizeof(unsigned), cudaMemcpyHostToDevice);

/* net.forward(d_inputs);*/
net.forward2(d_inputs);

cudaMemcpy(d_logits, net.get_output(), 1*10*sizeof(float), cudaMemcpyDeviceToDevice);
softmax_forward_loss_batch << <1, 1 >> >(d_logits, d_labels, d_prob, loss, 1, num_classes);

cudaStreamSynchronize(stream);

// ---- 打印中间输出 ----
cudaMemcpy(h_loss.data(), loss, 1*sizeof(float), cudaMemcpyDeviceToHost);
cudaMemcpy(h_prob.data(), d_prob, 1 * 10 * sizeof(float), cudaMemcpyDeviceToHost);

int ans = 0;
for (int i = 1; i < 10; i++)
if (h_prob[i] > h_prob[ans]) ans = i;//找到最大概率标号

if (h_labels[0] == ans)
last[epoch] = 1;

//找出最大概率是不是与标签同？同，则ok
//否则失败

///* float avg_loss = 0.f;
// for (float l : h_loss) avg_loss += l;
// avg_loss /= 1;*/
// if (epoch % 1 == 0) { // 每个 epoch 打印一次
// std::cout << "Count " << epoch
//
// << " 损失 = " << h_loss[0]
// << " max概率= " << h_prob[ans]
// << "max响应标签 " << ans
// << " (实际标签=" << h_labels[0] << ")"
// << std::endl;
// }
}
cudaFree(d_inputs);
cudaFree(d_labels);
cudaFree(d_logits);
cudaFree(d_prob);
cudaFree(loss);
cudaFree(dlogits);

float junzhi = 0;

int score = 0;
for (int i = 0; i < 10000; i++)
{
if (last[i] == 1)score++;
}
junzhi = score / (float)100;

std::cout << "test score= " << junzhi


<< std::endl;
return junzhi;
}

因为是x64，release实现，很难debug，最后改的一团糟！而且上面程序一直正常运行，当然刚开始，我不敢想其他问题，只能闷着头皮，找自己的问题！

春天出去溜达一圈，想了想，可不可以像训练一样批处理出成绩？！这个注意好！立马改：

void test2(int epochs, int batch_size, LeNet &net, cudaStream_t stream, cifar10DatasetTEST &Dataset) {
float *d_inputs;
unsigned *d_labels;
//输出的预测概率，softmax后的结果，损失，预测概率的梯度（作为LeNet反向传播的输入）
float *d_logits, *d_prob, *loss, *dlogits;
int num_classes = 10;
//cudaMalloc(&d_inputs, batch_size * 32*32 *3 * sizeof(float));
cudaMalloc(&d_inputs, batch_size * 32 * 32 * 5 * sizeof(float));
cudaMalloc(&d_labels, batch_size*sizeof(unsigned));
cudaMalloc(&d_logits, batch_size*num_classes*sizeof(float));
cudaMalloc(&d_prob, batch_size*num_classes*sizeof(float));
cudaMalloc(&loss, batch_size*sizeof(float));
cudaMalloc(&dlogits, batch_size*num_classes*sizeof(float));
std::vector<float> h_loss(batch_size);
int num_errorsTRain = 0;
auto t1 = std::chrono::high_resolution_clock::now();
for (int epoch = 0; epoch<epochs; epoch++) {
auto batch = Dataset.next_batch(batch_size);
std::vector<unsigned> h_labels(batch_size);
for (int batch_idx = 0; batch_idx<batch_size; batch_idx++) {
/*cudaMemcpy(d_inputs + batch_idx * 32*32 * 3, batch[batch_idx].image.data(), 32 * 32 * 3 * sizeof(float), cudaMemcpyHostToDevice);*/
cudaMemcpy(d_inputs + batch_idx * 32 * 32 * 5, batch[batch_idx].image.data(), 32 * 32 * 5 * sizeof(float), cudaMemcpyHostToDevice);
h_labels[batch_idx] = batch[batch_idx].label;
}
cudaMemcpy(d_labels, h_labels.data(),
batch_size*sizeof(unsigned), cudaMemcpyHostToDevice);
net.forward2(d_inputs);
cudaMemcpy(d_logits, net.get_output(), batch_size*num_classes*sizeof(float), cudaMemcpyDeviceToDevice);
softmax_forward_loss_batch << <batch_size, 1 >> >(d_logits, d_labels, d_prob, loss, batch_size, num_classes);

cudaStreamSynchronize(stream);

// ---- 打印中间输出 ----
std::vector<float> h_prob(10 * batch_size);
cudaMemcpy(h_prob.data(), d_prob, batch_size * 10 * sizeof(float), cudaMemcpyDeviceToHost);

for (int b = 0; b < batch_size; b++)
{
int ans = 0;
for (int i = 1; i < 10; i++)
if (h_prob[i + b * 10] > h_prob[ans + b * 10]) ans = i;//找到最大概率标号

if (h_labels[0 + b] == ans)
// last[epoch] = 1;
++num_errorsTRain;
}
//// ---- 打印中间输出 ----
//cudaMemcpy(h_loss.data(), loss, batch_size*sizeof(float), cudaMemcpyDeviceToHost);
//float avg_loss = 0.f;
//for (float l : h_loss) avg_loss += l;
//avg_loss /= batch_size;
////if (epoch % 1 == 0) { // 每个 epoch 打印一次
//// std::cout << "[Rank " << rank << "] Epoch " << epoch
//// << " Avg Loss = " << avg_loss
//// << " (first label=" << h_labels[0] << ")"
//// << std::endl;
////}

}
auto t2 = std::chrono::high_resolution_clock::now();
printf("时间: %f ms\n", std::chrono::duration_cast<std::chrono::microseconds>(t2 - t1).count() / 1000.0f);
printf("Test Classification result: %.2f%% ok (used %d images)\n", ((float)num_errorsTRain / (batch_size*epochs)) * 100.0f, (int)batch_size*epochs);

//float *Junzhi = get_u();
//float *fangcha = get_var();//运行输出值
//for (int jf = 0; jf < 32; jf++)
// printf("均值: %f,方差:%f \n", Junzhi[jf], fangcha[jf]);

cudaFree(d_inputs);
cudaFree(d_labels);
cudaFree(d_logits);
cudaFree(d_prob);
cudaFree(loss);
cudaFree(dlogits);
}

调用如下：

test2(156, batch_size, LeNet_net, stream, DatasetsTest);//样本10000/64batch=156轮

竟然测试ok了！这个是再mx550显卡上！

这是一种办法，另外我把源程序在另一台电脑1060显卡上运行！也是win10，cuda9.0，cudnn7.1.4，老办法，也成功了！但1060显卡用不了batch=64，训练时，改成batch=32，测试仍然用单样本！

说明什么问题？方法总比问题多！