28 #ifndef __itkTestDriverInclude_h
29 #define __itkTestDriverInclude_h
39 #include "itksys/Process.h"
55 #include "itksys/SystemTools.hxx"
60 #include "vnl/vnl_sample.h"
62 #define ITK_TEST_DIMENSION_MAX 6
65 const char *baselineImageFilename,
67 double intensityTolerance,
69 unsigned int radiusTolerance = 0);
72 const std::string md5hash );
91 typedef std::pair< const char *, std::vector<std::string> >
HashPairType;
122 std::cerr <<
"usage: itkTestDriver [options] prg [args]" << std::endl;
123 std::cerr <<
" itkTestDriver --no-process [options]" << std::endl;
124 std::cerr << std::endl;
125 std::cerr <<
"itkTestDriver alter the environment, run a test program and compare the images" << std::endl;
126 std::cerr <<
"produced." << std::endl;
127 std::cerr << std::endl;
128 std::cerr <<
"Options:" << std::endl;
129 std::cerr <<
" --add-before-libpath PATH" << std::endl;
130 std::cerr <<
" Add a path to the library path environment. This option take care of" << std::endl;
131 std::cerr <<
" choosing the right environment variable for your system." << std::endl;
132 std::cerr <<
" This option can be used several times." << std::endl;
133 std::cerr << std::endl;
134 std::cerr <<
" --add-before-env NAME VALUE" << std::endl;
135 std::cerr <<
" Add a VALUE to the variable name in the environment." << std::endl;
136 std::cerr <<
" The seperator used is the default one on the system." << std::endl;
137 std::cerr <<
" This option can be used several times." << std::endl;
138 std::cerr << std::endl;
139 std::cerr <<
" --add-before-env-with-sep NAME VALUE SEP" << std::endl;
140 std::cerr <<
" Add a VALUE to the variable name in the environment using the provided separator." << std::endl;
141 std::cerr <<
" This option can be used several times." << std::endl;
142 std::cerr << std::endl;
143 std::cerr <<
" --compare TEST BASELINE" << std::endl;
144 std::cerr <<
" Compare the TEST image to the BASELINE one." << std::endl;
145 std::cerr <<
" This option can be used several times." << std::endl;
146 std::cerr << std::endl;
147 std::cerr <<
" --compare-MD5 TEST md5hash0 [ md5hash1 ... ]" << std::endl;
148 std::cerr <<
" Compare the TEST image file's md5 hash to the provided hash." << std::endl;
149 std::cerr <<
" md5hash0 is required and assumed to be a hash." << std::endl;
150 std::cerr <<
" Additional arguments are considered hashes when the string is 32 hexi-decimal characters. " << std::endl;
151 std::cerr <<
" This option can be used several times for multiple comparisons." << std::endl;
152 std::cerr << std::endl;
153 std::cerr <<
" --with-threads THREADS" << std::endl;
154 std::cerr <<
" Use at most THREADS threads." << std::endl;
155 std::cerr << std::endl;
156 std::cerr <<
" --without-threads" << std::endl;
157 std::cerr <<
" Use at most one thread." << std::endl;
158 std::cerr << std::endl;
159 std::cerr <<
" --compareNumberOfPixelsTolerance TOLERANCE" << std::endl;
160 std::cerr <<
" When comparing images with --compare, allow TOLERANCE pixels to differ." << std::endl;
161 std::cerr <<
" Default is 0." << std::endl;
162 std::cerr << std::endl;
163 std::cerr <<
" --compareRadiusTolerance TOLERANCE" << std::endl;
164 std::cerr <<
" Default is 0." << std::endl;
165 std::cerr << std::endl;
166 std::cerr <<
" --compareIntensityTolerance TOLERANCE" << std::endl;
167 std::cerr <<
" Default is 2.0." << std::endl;
168 std::cerr << std::endl;
169 std::cerr <<
" --no-process" << std::endl;
170 std::cerr <<
" The test driver will not invoke any process." << std::endl;
171 std::cerr << std::endl;
172 std::cerr <<
" --full-output" << std::endl;
173 std::cerr <<
" Causes the full output of the test to be passed to cdash." << std::endl;
174 std::cerr <<
" --redirect-output TEST_OUTPUT" << std::endl;
175 std::cerr <<
" Redirects the test output to the file TEST_OUTPUT." << std::endl;
176 std::cerr << std::endl;
177 std::cerr <<
" --" << std::endl;
178 std::cerr <<
" The options after -- are not interpreted by this program and passed" << std::endl;
179 std::cerr <<
" directly to the test program." << std::endl;
180 std::cerr << std::endl;
181 std::cerr <<
" --help" << std::endl;
182 std::cerr <<
" Display this message and exit." << std::endl;
183 std::cerr << std::endl;
196 redirectOutputParameters.
redirect =
false;
198 if( processedOutput )
200 processedOutput->externalProcessMustBeCalled =
true;
208 if ( !skip && strcmp((*av)[i],
"--compare") == 0 )
219 else if ( !skip && strcmp((*av)[i],
"--compare-MD5") == 0 )
226 const char *filename = (*av)[i + 1];
227 std::string md5hash0 = (*av)[i + 2];
230 for( std::string::iterator iter = md5hash0.begin(); iter != md5hash0.end(); ++iter )
232 *iter = tolower(*iter);
236 if ( md5hash0.size() != 32 ||
237 md5hash0.find_first_not_of(
"0123456789abcdef" ) != std::string::npos )
239 std::cerr <<
"Warning: argument does not appear to be a valid md5 hash \"" << md5hash0 <<
"\"." << std::endl;
242 std::vector< std::string > hashVector;
243 hashVector.push_back( md5hash0 );
249 while ( *ac - i > 0 )
251 std::string md5hashN = (*av)[i];
254 for( std::string::iterator iter = md5hashN.begin(); iter != md5hashN.end(); ++iter )
256 *iter = tolower(*iter);
260 if ( md5hashN.size() != 32 ||
261 md5hashN.find_first_not_of(
"0123456789abcdef" ) != std::string::npos )
267 hashVector.push_back( md5hashN );
276 hashTestList.push_back(
HashPairType( filename, hashVector ) );
279 else if ( !skip && strcmp((*av)[i],
"--") == 0 )
284 else if ( !skip && strcmp((*av)[i],
"--help") == 0 )
289 else if ( !skip && strcmp((*av)[i],
"--with-threads") == 0 )
297 std::string threadEnv =
"ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=";
298 threadEnv += (*av)[i + 1];
299 itksys::SystemTools::PutEnv( threadEnv.c_str() );
301 itk::MultiThreader::SetGlobalDefaultNumberOfThreads(atoi((*av)[i + 1]));
305 else if ( !skip && strcmp((*av)[i],
"--without-threads") == 0 )
307 itksys::SystemTools::PutEnv(
"ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=1" );
308 itk::MultiThreader::SetGlobalDefaultNumberOfThreads(1);
312 else if ( !skip && strcmp((*av)[i],
"--compareNumberOfPixelsTolerance") == 0 )
323 else if ( !skip && strcmp((*av)[i],
"--compareRadiusTolerance") == 0 )
334 else if ( !skip && strcmp((*av)[i],
"--compareIntensityTolerance") == 0 )
345 else if ( !skip && strcmp((*av)[i],
"--add-before-libpath") == 0 )
352 if( processedOutput )
354 processedOutput->add_before_libpath.push_back( (*av)[i+1] );
359 else if ( !skip && strcmp((*av)[i],
"--add-before-env") == 0 )
366 if( processedOutput )
368 processedOutput->add_before_env.push_back( (*av)[i+1] );
369 processedOutput->add_before_env.push_back( (*av)[i+2] );
374 else if ( !skip && strcmp((*av)[i],
"--add-before-env-with-sep") == 0 )
381 if( processedOutput )
383 processedOutput->add_before_env_with_sep.push_back( (*av)[i+1] );
384 processedOutput->add_before_env_with_sep.push_back( (*av)[i+2] );
385 processedOutput->add_before_env_with_sep.push_back( (*av)[i+3] );
390 else if ( !skip && strcmp((*av)[i],
"--full-output") == 0 )
394 std::cout <<
"CTEST_FULL_OUTPUT" << std::endl;
398 else if ( !skip && strcmp((*av)[i],
"--no-process") == 0 )
402 if( processedOutput )
404 processedOutput->externalProcessMustBeCalled =
false;
409 else if ( !skip && strcmp((*av)[i],
"--redirectOutput") == 0 )
416 redirectOutputParameters.
redirect =
true;
417 redirectOutputParameters.
fileName = (*av)[i + 1];
423 if( processedOutput )
425 processedOutput->args.push_back((*av)[i]);
444 const char *baselineImageFilename,
446 double intensityTolerance,
448 unsigned int radiusTolerance)
458 ReaderType::Pointer baselineReader = ReaderType::New();
459 baselineReader->SetFileName(baselineImageFilename);
462 baselineReader->UpdateLargestPossibleRegion();
466 std::cerr <<
"Exception detected while reading " << baselineImageFilename <<
" : " << e.
GetDescription();
471 ReaderType::Pointer testReader = ReaderType::New();
472 testReader->SetFileName(testImageFilename);
475 testReader->UpdateLargestPossibleRegion();
479 std::cerr <<
"Exception detected while reading " << testImageFilename <<
" : " << e.
GetDescription() << std::endl;
484 ImageType::SizeType baselineSize;
485 baselineSize = baselineReader->GetOutput()->GetLargestPossibleRegion().GetSize();
486 ImageType::SizeType testSize;
487 testSize = testReader->GetOutput()->GetLargestPossibleRegion().GetSize();
489 if ( baselineSize != testSize )
491 std::cerr <<
"The size of the Baseline image and Test image do not match!" << std::endl;
492 std::cerr <<
"Baseline image: " << baselineImageFilename
493 <<
" has size " << baselineSize << std::endl;
494 std::cerr <<
"Test image: " << testImageFilename
495 <<
" has size " << testSize << std::endl;
501 DiffType::Pointer diff = DiffType::New();
502 diff->SetValidInput( baselineReader->GetOutput() );
503 diff->SetTestInput( testReader->GetOutput() );
504 diff->SetDifferenceThreshold(intensityTolerance);
505 diff->SetToleranceRadius(radiusTolerance);
506 diff->UpdateLargestPossibleRegion();
509 status = diff->GetNumberOfPixelsWithDifferences();
511 if ( ! reportErrors )
515 std::string shortFilename = itksys::SystemTools::GetFilenameName( baselineImageFilename );
517 std::cout <<
"<DartMeasurement name=\"ImageError " << shortFilename
518 <<
"\" type=\"numeric/double\">";
520 std::cout <<
"</DartMeasurement>" << std::endl;
524 if ( ( status > numberOfPixelsTolerance ) && reportErrors )
528 std::cout <<
"<DartMeasurement name=\"ImageError\" type=\"numeric/double\">";
530 std::cout <<
"</DartMeasurement>" << std::endl;
534 std::cout <<
"<DartMeasurement name=\"ImageError Minimum\" type=\"numeric/double\">";
535 std::cout << diff->GetMinimumDifference() <<
"</DartMeasurement>" << std::endl;
537 std::cout <<
"<DartMeasurement name=\"ImageError Maximum\" type=\"numeric/double\">";
538 std::cout << diff->GetMaximumDifference() <<
"</DartMeasurement>" << std::endl;
540 std::cout <<
"<DartMeasurement name=\"ImageError Mean\" type=\"numeric/double\">";
541 std::cout << diff->GetMeanDifference() <<
"</DartMeasurement>" << std::endl;
548 OutputType::SizeType size; size.Fill(0);
550 RescaleType::Pointer rescale = RescaleType::New();
553 rescale->SetInput( diff->GetOutput() );
554 rescale->UpdateLargestPossibleRegion();
555 size = rescale->GetOutput()->GetLargestPossibleRegion().GetSize();
559 OutputType::IndexType index; index.Fill(0);
562 index[i] = size[i] / 2;
568 region.SetIndex(index);
570 region.SetSize(size);
572 ExtractType::Pointer extract = ExtractType::New();
573 extract->SetDirectionCollapseToIdentity();
574 extract->SetInput( rescale->GetOutput() );
575 extract->SetExtractionRegion(region);
577 WriterType::Pointer writer = WriterType::New();
578 writer->SetInput( extract->GetOutput() );
580 std::ostringstream diffName;
581 diffName << testImageFilename <<
".diff.png";
584 rescale->SetInput( diff->GetOutput() );
587 catch (
const std::exception & e )
589 std::cerr <<
"Error during rescale of " << diffName.str() << std::endl;
590 std::cerr << e.what() <<
"\n";
594 std::cerr <<
"Error during rescale of " << diffName.str() << std::endl;
596 writer->SetFileName( diffName.str().c_str() );
601 catch (
const std::exception & e )
603 std::cerr <<
"Error during write of " << diffName.str() << std::endl;
604 std::cerr << e.what() <<
"\n";
608 std::cerr <<
"Error during write of " << diffName.str() << std::endl;
611 std::cout <<
"<DartMeasurementFile name=\"DifferenceImage\" type=\"image/png\">";
612 std::cout << diffName.str();
613 std::cout <<
"</DartMeasurementFile>" << std::endl;
615 std::ostringstream baseName;
616 baseName << testImageFilename <<
".base.png";
619 rescale->SetInput( baselineReader->GetOutput() );
622 catch (
const std::exception & e )
624 std::cerr <<
"Error during rescale of " << baseName.str() << std::endl;
625 std::cerr << e.what() <<
"\n";
629 std::cerr <<
"Error during rescale of " << baseName.str() << std::endl;
633 writer->SetFileName( baseName.str().c_str() );
636 catch (
const std::exception & e )
638 std::cerr <<
"Error during write of " << baseName.str() << std::endl;
639 std::cerr << e.what() <<
"\n";
643 std::cerr <<
"Error during write of " << baseName.str() << std::endl;
646 std::cout <<
"<DartMeasurementFile name=\"BaselineImage\" type=\"image/png\">";
647 std::cout << baseName.str();
648 std::cout <<
"</DartMeasurementFile>" << std::endl;
650 std::ostringstream testName;
651 testName << testImageFilename <<
".test.png";
654 rescale->SetInput( testReader->GetOutput() );
657 catch (
const std::exception & e )
659 std::cerr <<
"Error during rescale of " << testName.str() << std::endl;
660 std::cerr << e.what() <<
"\n";
664 std::cerr <<
"Error during rescale of " << testName.str() << std::endl;
668 writer->SetFileName( testName.str().c_str() );
671 catch (
const std::exception & e )
673 std::cerr <<
"Error during write of " << testName.str() << std::endl;
674 std::cerr << e.what() <<
"\n";
678 std::cerr <<
"Error during write of " << testName.str() << std::endl;
681 std::cout <<
"<DartMeasurementFile name=\"TestImage\" type=\"image/png\">";
682 std::cout << testName.str();
683 std::cout <<
"</DartMeasurementFile>" << std::endl;
685 return ( status > numberOfPixelsTolerance ) ? status : 0;
688 template<
typename TImageType >
691 typedef TImageType ImageType;
696 typename ReaderType::Pointer testReader = ReaderType::New();
697 testReader->SetFileName(testImageFilename);
700 testReader->UpdateLargestPossibleRegion();
704 std::cerr <<
"Exception detected while reading " << testImageFilename <<
" : " << e.
GetDescription() << std::endl;
710 typename HashFilterType::Pointer hasher = HashFilterType::New();
711 hasher->SetInput( testReader->GetOutput() );
714 return hasher->GetHash();
719 const std::vector<std::string> &baselineMD5Vector )
726 itkGenericExceptionMacro(
"Unable to determine ImageIO reader for \"" << testImageFilename <<
"\"" );
730 iobase->SetFileName( testImageFilename );
731 iobase->ReadImageInformation();
736 std::string testMD5 =
"";
737 switch(componentType)
740 testMD5 = ComputeHash< itk::VectorImage<char, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
743 testMD5 = ComputeHash< itk::VectorImage<unsigned char, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
746 testMD5 = ComputeHash< itk::VectorImage<short, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
749 testMD5 = ComputeHash< itk::VectorImage<unsigned short, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
752 testMD5 = ComputeHash< itk::VectorImage<int, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
755 testMD5 = ComputeHash< itk::VectorImage<unsigned int, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
758 testMD5 = ComputeHash< itk::VectorImage<long, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
761 testMD5 = ComputeHash< itk::VectorImage<unsigned long, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
765 std::cerr <<
"Hashing is not supporting for float and double images." << std::endl;
766 itkGenericExceptionMacro(
"Hashing is not supported for images of float or doubles." );
771 itkGenericExceptionMacro(
"Logic error!" );
774 std::vector<std::string>::const_iterator iter = baselineMD5Vector.begin();
775 assert( baselineMD5Vector.size() );
778 if ( *iter == testMD5 )
784 while (++iter != baselineMD5Vector.end() );
787 std::cout <<
"<DartMeasurement name=\"TestMD5\" type=\"text/string\">";
788 std::cout << testMD5;
789 std::cout <<
"</DartMeasurement>" << std::endl;
793 for ( iter = baselineMD5Vector.begin(); iter != baselineMD5Vector.end(); ++iter )
795 std::cout <<
"<DartMeasurement name=\"BaselineMD5\" type=\"text/string\">";
797 std::cout <<
"</DartMeasurement>" << std::endl;
809 ReaderType::Pointer reader = ReaderType::New();
810 reader->SetFileName( testImageFilename );
811 reader->UpdateLargestPossibleRegion();
813 ImageType::SizeType size;
814 size = reader->GetOutput()->GetLargestPossibleRegion().GetSize();
818 ImageType::IndexType index; index.Fill(0);
821 index[i] = size[i] / 2;
827 ImageType::RegionType region;
828 region.SetIndex(index);
830 region.SetSize(size);
832 ExtractType::Pointer extract = ExtractType::New();
833 extract->SetDirectionCollapseToIdentity();
834 extract->SetInput( reader->GetOutput() );
835 extract->SetExtractionRegion(region);
837 RescaleType::Pointer rescale = RescaleType::New();
840 rescale->SetInput( extract->GetOutput() );
842 WriterType::Pointer writer = WriterType::New();
843 writer->SetInput( rescale->GetOutput() );
846 std::ostringstream testName;
847 testName << testImageFilename <<
".test.png";
849 writer->SetFileName( testName.str().c_str() );
853 rescale->UpdateLargestPossibleRegion();
856 catch (
const std::exception &
e )
858 std::cerr <<
"Error during rescale and writing of " << testName.str()<< std::endl;
859 std::cerr << e.what() <<
"\n";
863 std::cerr <<
"Unknow error during rescale and writing of " << testName.str() << std::endl;
866 std::cout <<
"<DartMeasurementFile name=\"TestImage\" type=\"image/png\">";
867 std::cout << testName.str();
868 std::cout <<
"</DartMeasurementFile>" << std::endl;
884 std::map< std::string, int > baselines;
885 baselines[std::string(baselineFilename)] = 0;
887 std::string originalBaseline(baselineFilename);
890 std::string::size_type suffixPos = originalBaseline.rfind(
".");
892 if ( suffixPos != std::string::npos )
894 suffix = originalBaseline.substr( suffixPos, originalBaseline.length() );
895 originalBaseline.erase( suffixPos, originalBaseline.length() );
899 std::ostringstream filename;
900 filename << originalBaseline <<
"." << x << suffix;
901 std::ifstream filestream( filename.str().c_str() );
906 baselines[filename.str()] = 0;
914 #include "itkTestingComparisonImageFilter.hxx"