28 #ifndef __itkTestDriverInclude_h
29 #define __itkTestDriverInclude_h
39 #include "itksys/Process.h"
55 #include "itksys/SystemTools.hxx"
60 #include "vnl/vnl_sample.h"
62 #define ITK_TEST_DIMENSION_MAX 6
65 const char *baselineImageFilename,
67 double intensityTolerance,
69 unsigned int radiusTolerance = 0);
72 const std::string md5hash );
91 typedef std::pair< const char *, std::vector<std::string> >
HashPairType;
122 std::cerr <<
"usage: itkTestDriver [options] prg [args]" << std::endl;
123 std::cerr <<
" itkTestDriver --no-process [options]" << std::endl;
124 std::cerr << std::endl;
125 std::cerr <<
"itkTestDriver alter the environment, run a test program and compare the images" << std::endl;
126 std::cerr <<
"produced." << std::endl;
127 std::cerr << std::endl;
128 std::cerr <<
"Options:" << std::endl;
129 std::cerr <<
" --add-before-libpath PATH" << std::endl;
130 std::cerr <<
" Add a path to the library path environment. This option take care of" << std::endl;
131 std::cerr <<
" choosing the right environment variable for your system." << std::endl;
132 std::cerr <<
" This option can be used several times." << std::endl;
133 std::cerr << std::endl;
134 std::cerr <<
" --add-before-env NAME VALUE" << std::endl;
135 std::cerr <<
" Add a VALUE to the variable name in the environment." << std::endl;
136 std::cerr <<
" The seperator used is the default one on the system." << std::endl;
137 std::cerr <<
" This option can be used several times." << std::endl;
138 std::cerr << std::endl;
139 std::cerr <<
" --add-before-env-with-sep NAME VALUE SEP" << std::endl;
140 std::cerr <<
" Add a VALUE to the variable name in the environment using the provided separator." << std::endl;
141 std::cerr <<
" This option can be used several times." << std::endl;
142 std::cerr << std::endl;
143 std::cerr <<
" --remove-env NAME" << std::endl;
144 std::cerr <<
" Remove the variable name from the environment." << std::endl;
145 std::cerr <<
" This option can be used several times." << std::endl;
146 std::cerr << std::endl;
147 std::cerr <<
" --compare TEST BASELINE" << std::endl;
148 std::cerr <<
" Compare the TEST image to the BASELINE one." << std::endl;
149 std::cerr <<
" This option can be used several times." << std::endl;
150 std::cerr << std::endl;
151 std::cerr <<
" --compare-MD5 TEST md5hash0 [ md5hash1 ... ]" << std::endl;
152 std::cerr <<
" Compare the TEST image file's md5 hash to the provided hash." << std::endl;
153 std::cerr <<
" md5hash0 is required and assumed to be a hash." << std::endl;
154 std::cerr <<
" Additional arguments are considered hashes when the string is 32 hexi-decimal characters. " << std::endl;
155 std::cerr <<
" This option can be used several times for multiple comparisons." << std::endl;
156 std::cerr << std::endl;
157 std::cerr <<
" --with-threads THREADS" << std::endl;
158 std::cerr <<
" Use at most THREADS threads." << std::endl;
159 std::cerr << std::endl;
160 std::cerr <<
" --without-threads" << std::endl;
161 std::cerr <<
" Use at most one thread." << std::endl;
162 std::cerr << std::endl;
163 std::cerr <<
" --compareNumberOfPixelsTolerance TOLERANCE" << std::endl;
164 std::cerr <<
" When comparing images with --compare, allow TOLERANCE pixels to differ." << std::endl;
165 std::cerr <<
" Default is 0." << std::endl;
166 std::cerr << std::endl;
167 std::cerr <<
" --compareRadiusTolerance TOLERANCE" << std::endl;
168 std::cerr <<
" Default is 0." << std::endl;
169 std::cerr << std::endl;
170 std::cerr <<
" --compareIntensityTolerance TOLERANCE" << std::endl;
171 std::cerr <<
" Default is 2.0." << std::endl;
172 std::cerr << std::endl;
173 std::cerr <<
" --no-process" << std::endl;
174 std::cerr <<
" The test driver will not invoke any process." << std::endl;
175 std::cerr << std::endl;
176 std::cerr <<
" --full-output" << std::endl;
177 std::cerr <<
" Causes the full output of the test to be passed to cdash." << std::endl;
178 std::cerr <<
" --redirect-output TEST_OUTPUT" << std::endl;
179 std::cerr <<
" Redirects the test output to the file TEST_OUTPUT." << std::endl;
180 std::cerr << std::endl;
181 std::cerr <<
" --" << std::endl;
182 std::cerr <<
" The options after -- are not interpreted by this program and passed" << std::endl;
183 std::cerr <<
" directly to the test program." << std::endl;
184 std::cerr << std::endl;
185 std::cerr <<
" --help" << std::endl;
186 std::cerr <<
" Display this message and exit." << std::endl;
187 std::cerr << std::endl;
200 redirectOutputParameters.
redirect =
false;
202 if( processedOutput )
204 processedOutput->externalProcessMustBeCalled =
true;
212 if ( !skip && strcmp((*av)[i],
"--compare") == 0 )
223 else if ( !skip && strcmp((*av)[i],
"--compare-MD5") == 0 )
230 const char *filename = (*av)[i + 1];
231 std::string md5hash0 = (*av)[i + 2];
234 for( std::string::iterator iter = md5hash0.begin(); iter != md5hash0.end(); ++iter )
236 *iter = tolower(*iter);
240 if ( md5hash0.size() != 32 ||
241 md5hash0.find_first_not_of(
"0123456789abcdef" ) != std::string::npos )
243 std::cerr <<
"Warning: argument does not appear to be a valid md5 hash \"" << md5hash0 <<
"\"." << std::endl;
246 std::vector< std::string > hashVector;
247 hashVector.push_back( md5hash0 );
253 while ( *ac - i > 0 )
255 std::string md5hashN = (*av)[i];
258 for( std::string::iterator iter = md5hashN.begin(); iter != md5hashN.end(); ++iter )
260 *iter = tolower(*iter);
264 if ( md5hashN.size() != 32 ||
265 md5hashN.find_first_not_of(
"0123456789abcdef" ) != std::string::npos )
271 hashVector.push_back( md5hashN );
280 hashTestList.push_back(
HashPairType( filename, hashVector ) );
283 else if ( !skip && strcmp((*av)[i],
"--") == 0 )
288 else if ( !skip && strcmp((*av)[i],
"--help") == 0 )
293 else if ( !skip && strcmp((*av)[i],
"--with-threads") == 0 )
301 std::string threadEnv =
"ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=";
302 threadEnv += (*av)[i + 1];
303 itksys::SystemTools::PutEnv( threadEnv.c_str() );
305 itk::MultiThreader::SetGlobalDefaultNumberOfThreads(atoi((*av)[i + 1]));
309 else if ( !skip && strcmp((*av)[i],
"--without-threads") == 0 )
311 itksys::SystemTools::PutEnv(
"ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=1" );
312 itk::MultiThreader::SetGlobalDefaultNumberOfThreads(1);
316 else if ( !skip && strcmp((*av)[i],
"--compareNumberOfPixelsTolerance") == 0 )
327 else if ( !skip && strcmp((*av)[i],
"--compareRadiusTolerance") == 0 )
338 else if ( !skip && strcmp((*av)[i],
"--compareIntensityTolerance") == 0 )
349 else if ( !skip && strcmp((*av)[i],
"--add-before-libpath") == 0 )
356 if( processedOutput )
358 processedOutput->add_before_libpath.push_back( (*av)[i+1] );
363 else if ( !skip && strcmp((*av)[i],
"--add-before-env") == 0 )
370 if( processedOutput )
372 processedOutput->add_before_env.push_back( (*av)[i+1] );
373 processedOutput->add_before_env.push_back( (*av)[i+2] );
378 else if ( !skip && strcmp((*av)[i],
"--add-before-env-with-sep") == 0 )
385 if( processedOutput )
387 processedOutput->add_before_env_with_sep.push_back( (*av)[i+1] );
388 processedOutput->add_before_env_with_sep.push_back( (*av)[i+2] );
389 processedOutput->add_before_env_with_sep.push_back( (*av)[i+3] );
394 else if ( !skip && strcmp((*av)[i],
"--remove-env") == 0 )
402 itksys::SystemTools::UnPutEnv( (*av)[i+1] );
407 else if ( !skip && strcmp((*av)[i],
"--full-output") == 0 )
411 std::cout <<
"CTEST_FULL_OUTPUT" << std::endl;
415 else if ( !skip && strcmp((*av)[i],
"--no-process") == 0 )
419 if( processedOutput )
421 processedOutput->externalProcessMustBeCalled =
false;
426 else if ( !skip && strcmp((*av)[i],
"--redirectOutput") == 0 )
433 redirectOutputParameters.
redirect =
true;
434 redirectOutputParameters.
fileName = (*av)[i + 1];
440 if( processedOutput )
442 processedOutput->args.push_back((*av)[i]);
462 const char *baselineImageFilename,
464 double intensityTolerance,
466 unsigned int radiusTolerance)
476 ReaderType::Pointer baselineReader = ReaderType::New();
477 baselineReader->SetFileName(baselineImageFilename);
480 baselineReader->UpdateLargestPossibleRegion();
484 std::cerr <<
"Exception detected while reading " << baselineImageFilename <<
" : " << e.
GetDescription();
489 ReaderType::Pointer testReader = ReaderType::New();
490 testReader->SetFileName(testImageFilename);
493 testReader->UpdateLargestPossibleRegion();
497 std::cerr <<
"Exception detected while reading " << testImageFilename <<
" : " << e.
GetDescription() << std::endl;
502 ImageType::SizeType baselineSize;
503 baselineSize = baselineReader->GetOutput()->GetLargestPossibleRegion().GetSize();
504 ImageType::SizeType testSize;
505 testSize = testReader->GetOutput()->GetLargestPossibleRegion().GetSize();
507 if ( baselineSize != testSize )
509 std::cerr <<
"The size of the Baseline image and Test image do not match!" << std::endl;
510 std::cerr <<
"Baseline image: " << baselineImageFilename
511 <<
" has size " << baselineSize << std::endl;
512 std::cerr <<
"Test image: " << testImageFilename
513 <<
" has size " << testSize << std::endl;
519 DiffType::Pointer diff = DiffType::New();
520 diff->SetValidInput( baselineReader->GetOutput() );
521 diff->SetTestInput( testReader->GetOutput() );
522 diff->SetDifferenceThreshold(intensityTolerance);
523 diff->SetToleranceRadius(radiusTolerance);
524 diff->UpdateLargestPossibleRegion();
527 status = diff->GetNumberOfPixelsWithDifferences();
529 if ( ! reportErrors )
533 std::string shortFilename = itksys::SystemTools::GetFilenameName( baselineImageFilename );
535 std::cout <<
"<DartMeasurement name=\"ImageError " << shortFilename
536 <<
"\" type=\"numeric/double\">";
538 std::cout <<
"</DartMeasurement>" << std::endl;
542 if ( ( status > numberOfPixelsTolerance ) && reportErrors )
546 std::cout <<
"<DartMeasurement name=\"ImageError\" type=\"numeric/double\">";
548 std::cout <<
"</DartMeasurement>" << std::endl;
552 std::cout <<
"<DartMeasurement name=\"ImageError Minimum\" type=\"numeric/double\">";
553 std::cout << diff->GetMinimumDifference() <<
"</DartMeasurement>" << std::endl;
555 std::cout <<
"<DartMeasurement name=\"ImageError Maximum\" type=\"numeric/double\">";
556 std::cout << diff->GetMaximumDifference() <<
"</DartMeasurement>" << std::endl;
558 std::cout <<
"<DartMeasurement name=\"ImageError Mean\" type=\"numeric/double\">";
559 std::cout << diff->GetMeanDifference() <<
"</DartMeasurement>" << std::endl;
566 OutputType::SizeType size; size.Fill(0);
568 RescaleType::Pointer rescale = RescaleType::New();
571 rescale->SetInput( diff->GetOutput() );
572 rescale->UpdateLargestPossibleRegion();
573 size = rescale->GetOutput()->GetLargestPossibleRegion().GetSize();
577 OutputType::IndexType index; index.Fill(0);
580 index[i] = size[i] / 2;
586 region.SetIndex(index);
588 region.SetSize(size);
590 ExtractType::Pointer extract = ExtractType::New();
591 extract->SetDirectionCollapseToIdentity();
592 extract->SetInput( rescale->GetOutput() );
593 extract->SetExtractionRegion(region);
595 WriterType::Pointer writer = WriterType::New();
596 writer->SetInput( extract->GetOutput() );
598 std::ostringstream diffName;
599 diffName << testImageFilename <<
".diff.png";
602 rescale->SetInput( diff->GetOutput() );
605 catch (
const std::exception & e )
607 std::cerr <<
"Error during rescale of " << diffName.str() << std::endl;
608 std::cerr << e.what() <<
"\n";
612 std::cerr <<
"Error during rescale of " << diffName.str() << std::endl;
614 writer->SetFileName( diffName.str().c_str() );
619 catch (
const std::exception & e )
621 std::cerr <<
"Error during write of " << diffName.str() << std::endl;
622 std::cerr << e.what() <<
"\n";
626 std::cerr <<
"Error during write of " << diffName.str() << std::endl;
629 std::cout <<
"<DartMeasurementFile name=\"DifferenceImage\" type=\"image/png\">";
630 std::cout << diffName.str();
631 std::cout <<
"</DartMeasurementFile>" << std::endl;
633 std::ostringstream baseName;
634 baseName << testImageFilename <<
".base.png";
637 rescale->SetInput( baselineReader->GetOutput() );
640 catch (
const std::exception & e )
642 std::cerr <<
"Error during rescale of " << baseName.str() << std::endl;
643 std::cerr << e.what() <<
"\n";
647 std::cerr <<
"Error during rescale of " << baseName.str() << std::endl;
651 writer->SetFileName( baseName.str().c_str() );
654 catch (
const std::exception & e )
656 std::cerr <<
"Error during write of " << baseName.str() << std::endl;
657 std::cerr << e.what() <<
"\n";
661 std::cerr <<
"Error during write of " << baseName.str() << std::endl;
664 std::cout <<
"<DartMeasurementFile name=\"BaselineImage\" type=\"image/png\">";
665 std::cout << baseName.str();
666 std::cout <<
"</DartMeasurementFile>" << std::endl;
668 std::ostringstream testName;
669 testName << testImageFilename <<
".test.png";
672 rescale->SetInput( testReader->GetOutput() );
675 catch (
const std::exception & e )
677 std::cerr <<
"Error during rescale of " << testName.str() << std::endl;
678 std::cerr << e.what() <<
"\n";
682 std::cerr <<
"Error during rescale of " << testName.str() << std::endl;
686 writer->SetFileName( testName.str().c_str() );
689 catch (
const std::exception & e )
691 std::cerr <<
"Error during write of " << testName.str() << std::endl;
692 std::cerr << e.what() <<
"\n";
696 std::cerr <<
"Error during write of " << testName.str() << std::endl;
699 std::cout <<
"<DartMeasurementFile name=\"TestImage\" type=\"image/png\">";
700 std::cout << testName.str();
701 std::cout <<
"</DartMeasurementFile>" << std::endl;
703 return ( status > numberOfPixelsTolerance ) ? status : 0;
706 template<
typename TImageType >
709 typedef TImageType ImageType;
714 typename ReaderType::Pointer testReader = ReaderType::New();
715 testReader->SetFileName(testImageFilename);
718 testReader->UpdateLargestPossibleRegion();
722 std::cerr <<
"Exception detected while reading " << testImageFilename <<
" : " << e.
GetDescription() << std::endl;
728 typename HashFilterType::Pointer hasher = HashFilterType::New();
729 hasher->SetInput( testReader->GetOutput() );
732 return hasher->GetHash();
737 const std::vector<std::string> &baselineMD5Vector )
744 itkGenericExceptionMacro(
"Unable to determine ImageIO reader for \"" << testImageFilename <<
"\"" );
748 iobase->SetFileName( testImageFilename );
749 iobase->ReadImageInformation();
754 std::string testMD5 =
"";
755 switch(componentType)
758 testMD5 = ComputeHash< itk::VectorImage<char, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
761 testMD5 = ComputeHash< itk::VectorImage<unsigned char, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
764 testMD5 = ComputeHash< itk::VectorImage<short, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
767 testMD5 = ComputeHash< itk::VectorImage<unsigned short, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
770 testMD5 = ComputeHash< itk::VectorImage<int, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
773 testMD5 = ComputeHash< itk::VectorImage<unsigned int, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
776 testMD5 = ComputeHash< itk::VectorImage<long, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
779 testMD5 = ComputeHash< itk::VectorImage<unsigned long, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
783 std::cerr <<
"Hashing is not supporting for float and double images." << std::endl;
784 itkGenericExceptionMacro(
"Hashing is not supported for images of float or doubles." );
789 itkGenericExceptionMacro(
"Logic error!" );
792 std::vector<std::string>::const_iterator iter = baselineMD5Vector.begin();
793 assert( baselineMD5Vector.size() );
796 if ( *iter == testMD5 )
802 while (++iter != baselineMD5Vector.end() );
805 std::cout <<
"<DartMeasurement name=\"TestMD5\" type=\"text/string\">";
806 std::cout << testMD5;
807 std::cout <<
"</DartMeasurement>" << std::endl;
811 for ( iter = baselineMD5Vector.begin(); iter != baselineMD5Vector.end(); ++iter )
813 std::cout <<
"<DartMeasurement name=\"BaselineMD5\" type=\"text/string\">";
815 std::cout <<
"</DartMeasurement>" << std::endl;
827 ReaderType::Pointer reader = ReaderType::New();
828 reader->SetFileName( testImageFilename );
829 reader->UpdateLargestPossibleRegion();
831 ImageType::SizeType size;
832 size = reader->GetOutput()->GetLargestPossibleRegion().GetSize();
836 ImageType::IndexType index; index.Fill(0);
839 index[i] = size[i] / 2;
845 ImageType::RegionType region;
846 region.SetIndex(index);
848 region.SetSize(size);
850 ExtractType::Pointer extract = ExtractType::New();
851 extract->SetDirectionCollapseToIdentity();
852 extract->SetInput( reader->GetOutput() );
853 extract->SetExtractionRegion(region);
855 RescaleType::Pointer rescale = RescaleType::New();
858 rescale->SetInput( extract->GetOutput() );
860 WriterType::Pointer writer = WriterType::New();
861 writer->SetInput( rescale->GetOutput() );
864 std::ostringstream testName;
865 testName << testImageFilename <<
".test.png";
867 writer->SetFileName( testName.str().c_str() );
871 rescale->UpdateLargestPossibleRegion();
874 catch (
const std::exception &
e )
876 std::cerr <<
"Error during rescale and writing of " << testName.str()<< std::endl;
877 std::cerr << e.what() <<
"\n";
881 std::cerr <<
"Unknow error during rescale and writing of " << testName.str() << std::endl;
884 std::cout <<
"<DartMeasurementFile name=\"TestImage\" type=\"image/png\">";
885 std::cout << testName.str();
886 std::cout <<
"</DartMeasurementFile>" << std::endl;
902 std::map< std::string, int > baselines;
903 baselines[std::string(baselineFilename)] = 0;
905 std::string originalBaseline(baselineFilename);
908 std::string::size_type suffixPos = originalBaseline.rfind(
".");
910 if ( suffixPos != std::string::npos )
912 suffix = originalBaseline.substr( suffixPos, originalBaseline.length() );
913 originalBaseline.erase( suffixPos, originalBaseline.length() );
917 std::ostringstream filename;
918 filename << originalBaseline <<
"." << x << suffix;
919 std::ifstream filestream( filename.str().c_str() );
924 baselines[filename.str()] = 0;
932 #include "itkTestingComparisonImageFilter.hxx"