28 #ifndef __itkTestDriverInclude_h
29 #define __itkTestDriverInclude_h
39 #include "itksys/Process.h"
56 #include "itksys/SystemTools.hxx"
61 #include "vnl/vnl_sample.h"
63 #define ITK_TEST_DIMENSION_MAX 6
66 const char *baselineImageFilename,
68 double intensityTolerance,
70 unsigned int radiusTolerance = 0);
73 const std::string md5hash );
92 typedef std::pair< const char *, std::vector<std::string> >
HashPairType;
123 std::cerr <<
"usage: itkTestDriver [options] prg [args]" << std::endl;
124 std::cerr <<
" itkTestDriver --no-process [options]" << std::endl;
125 std::cerr << std::endl;
126 std::cerr <<
"itkTestDriver alter the environment, run a test program and compare the images" << std::endl;
127 std::cerr <<
"produced." << std::endl;
128 std::cerr << std::endl;
129 std::cerr <<
"Options:" << std::endl;
130 std::cerr <<
" --add-before-libpath PATH" << std::endl;
131 std::cerr <<
" Add a path to the library path environment. This option take care of" << std::endl;
132 std::cerr <<
" choosing the right environment variable for your system." << std::endl;
133 std::cerr <<
" This option can be used several times." << std::endl;
134 std::cerr << std::endl;
135 std::cerr <<
" --add-before-env NAME VALUE" << std::endl;
136 std::cerr <<
" Add a VALUE to the variable name in the environment." << std::endl;
137 std::cerr <<
" The seperator used is the default one on the system." << std::endl;
138 std::cerr <<
" This option can be used several times." << std::endl;
139 std::cerr << std::endl;
140 std::cerr <<
" --add-before-env-with-sep NAME VALUE SEP" << std::endl;
141 std::cerr <<
" Add a VALUE to the variable name in the environment using the provided separator." << std::endl;
142 std::cerr <<
" This option can be used several times." << std::endl;
143 std::cerr << std::endl;
144 std::cerr <<
" --remove-env NAME" << std::endl;
145 std::cerr <<
" Remove the variable name from the environment." << std::endl;
146 std::cerr <<
" This option can be used several times." << std::endl;
147 std::cerr << std::endl;
148 std::cerr <<
" --compare TEST BASELINE" << std::endl;
149 std::cerr <<
" Compare the TEST image to the BASELINE one." << std::endl;
150 std::cerr <<
" This option can be used several times." << std::endl;
151 std::cerr << std::endl;
152 std::cerr <<
" --compare-MD5 TEST md5hash0 [ md5hash1 ... ]" << std::endl;
153 std::cerr <<
" Compare the TEST image file's md5 hash to the provided hash." << std::endl;
154 std::cerr <<
" md5hash0 is required and assumed to be a hash." << std::endl;
155 std::cerr <<
" Additional arguments are considered hashes when the string is 32 hexi-decimal characters. " << std::endl;
156 std::cerr <<
" This option can be used several times for multiple comparisons." << std::endl;
157 std::cerr << std::endl;
158 std::cerr <<
" --with-threads THREADS" << std::endl;
159 std::cerr <<
" Use at most THREADS threads." << std::endl;
160 std::cerr << std::endl;
161 std::cerr <<
" --without-threads" << std::endl;
162 std::cerr <<
" Use at most one thread." << std::endl;
163 std::cerr << std::endl;
164 std::cerr <<
" --compareNumberOfPixelsTolerance TOLERANCE" << std::endl;
165 std::cerr <<
" When comparing images with --compare, allow TOLERANCE pixels to differ." << std::endl;
166 std::cerr <<
" Default is 0." << std::endl;
167 std::cerr << std::endl;
168 std::cerr <<
" --compareRadiusTolerance TOLERANCE" << std::endl;
169 std::cerr <<
" Default is 0." << std::endl;
170 std::cerr << std::endl;
171 std::cerr <<
" --compareIntensityTolerance TOLERANCE" << std::endl;
172 std::cerr <<
" Default is 2.0." << std::endl;
173 std::cerr << std::endl;
174 std::cerr <<
" --no-process" << std::endl;
175 std::cerr <<
" The test driver will not invoke any process." << std::endl;
176 std::cerr << std::endl;
177 std::cerr <<
" --full-output" << std::endl;
178 std::cerr <<
" Causes the full output of the test to be passed to cdash." << std::endl;
179 std::cerr <<
" --redirect-output TEST_OUTPUT" << std::endl;
180 std::cerr <<
" Redirects the test output to the file TEST_OUTPUT." << std::endl;
181 std::cerr << std::endl;
182 std::cerr <<
" --" << std::endl;
183 std::cerr <<
" The options after -- are not interpreted by this program and passed" << std::endl;
184 std::cerr <<
" directly to the test program." << std::endl;
185 std::cerr << std::endl;
186 std::cerr <<
" --help" << std::endl;
187 std::cerr <<
" Display this message and exit." << std::endl;
188 std::cerr << std::endl;
193 return static_cast<char>( ::tolower(c));
205 redirectOutputParameters.
redirect =
false;
207 if( processedOutput )
209 processedOutput->externalProcessMustBeCalled =
true;
217 if ( !skip && strcmp((*av)[i],
"--compare") == 0 )
228 else if ( !skip && strcmp((*av)[i],
"--compare-MD5") == 0 )
235 const char *filename = (*av)[i + 1];
236 std::string md5hash0 = (*av)[i + 2];
239 std::transform(md5hash0.begin(), md5hash0.end(), md5hash0.begin(),
my_to_lower );
242 if ( md5hash0.size() != 32 ||
243 md5hash0.find_first_not_of(
"0123456789abcdef" ) != std::string::npos )
245 std::cerr <<
"Warning: argument does not appear to be a valid md5 hash \"" << md5hash0 <<
"\"." << std::endl;
248 std::vector< std::string > hashVector;
249 hashVector.push_back( md5hash0 );
255 while ( *ac - i > 0 )
257 std::string md5hashN = (*av)[i];
260 std::transform(md5hashN.begin(), md5hashN.end(), md5hashN.begin(),
my_to_lower );
263 if ( md5hashN.size() != 32 ||
264 md5hashN.find_first_not_of(
"0123456789abcdef" ) != std::string::npos )
270 hashVector.push_back( md5hashN );
279 hashTestList.push_back(
HashPairType( filename, hashVector ) );
282 else if ( !skip && strcmp((*av)[i],
"--") == 0 )
287 else if ( !skip && strcmp((*av)[i],
"--help") == 0 )
292 else if ( !skip && strcmp((*av)[i],
"--with-threads") == 0 )
300 std::string threadEnv =
"ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=";
301 threadEnv += (*av)[i + 1];
302 itksys::SystemTools::PutEnv( threadEnv.c_str() );
304 itk::MultiThreader::SetGlobalDefaultNumberOfThreads(atoi((*av)[i + 1]));
308 else if ( !skip && strcmp((*av)[i],
"--without-threads") == 0 )
310 itksys::SystemTools::PutEnv(
"ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=1" );
311 itk::MultiThreader::SetGlobalDefaultNumberOfThreads(1);
315 else if ( !skip && strcmp((*av)[i],
"--compareNumberOfPixelsTolerance") == 0 )
326 else if ( !skip && strcmp((*av)[i],
"--compareRadiusTolerance") == 0 )
337 else if ( !skip && strcmp((*av)[i],
"--compareIntensityTolerance") == 0 )
348 else if ( !skip && strcmp((*av)[i],
"--add-before-libpath") == 0 )
355 if( processedOutput )
357 processedOutput->add_before_libpath.push_back( (*av)[i+1] );
362 else if ( !skip && strcmp((*av)[i],
"--add-before-env") == 0 )
369 if( processedOutput )
371 processedOutput->add_before_env.push_back( (*av)[i+1] );
372 processedOutput->add_before_env.push_back( (*av)[i+2] );
377 else if ( !skip && strcmp((*av)[i],
"--add-before-env-with-sep") == 0 )
384 if( processedOutput )
386 processedOutput->add_before_env_with_sep.push_back( (*av)[i+1] );
387 processedOutput->add_before_env_with_sep.push_back( (*av)[i+2] );
388 processedOutput->add_before_env_with_sep.push_back( (*av)[i+3] );
393 else if ( !skip && strcmp((*av)[i],
"--remove-env") == 0 )
401 itksys::SystemTools::UnPutEnv( (*av)[i+1] );
406 else if ( !skip && strcmp((*av)[i],
"--full-output") == 0 )
410 std::cout <<
"CTEST_FULL_OUTPUT" << std::endl;
414 else if ( !skip && strcmp((*av)[i],
"--no-process") == 0 )
418 if( processedOutput )
420 processedOutput->externalProcessMustBeCalled =
false;
425 else if ( !skip && strcmp((*av)[i],
"--redirectOutput") == 0 )
432 redirectOutputParameters.
redirect =
true;
433 redirectOutputParameters.
fileName = (*av)[i + 1];
439 if( processedOutput )
441 processedOutput->args.push_back((*av)[i]);
461 const char *baselineImageFilename,
463 double intensityTolerance,
465 unsigned int radiusTolerance)
475 ReaderType::Pointer baselineReader = ReaderType::New();
476 baselineReader->SetFileName(baselineImageFilename);
479 baselineReader->UpdateLargestPossibleRegion();
483 std::cerr <<
"Exception detected while reading " << baselineImageFilename <<
" : " << e.
GetDescription();
488 ReaderType::Pointer testReader = ReaderType::New();
489 testReader->SetFileName(testImageFilename);
492 testReader->UpdateLargestPossibleRegion();
496 std::cerr <<
"Exception detected while reading " << testImageFilename <<
" : " << e.
GetDescription() << std::endl;
501 ImageType::SizeType baselineSize;
502 baselineSize = baselineReader->GetOutput()->GetLargestPossibleRegion().GetSize();
503 ImageType::SizeType testSize;
504 testSize = testReader->GetOutput()->GetLargestPossibleRegion().GetSize();
506 if ( baselineSize != testSize )
508 std::cerr <<
"The size of the Baseline image and Test image do not match!" << std::endl;
509 std::cerr <<
"Baseline image: " << baselineImageFilename
510 <<
" has size " << baselineSize << std::endl;
511 std::cerr <<
"Test image: " << testImageFilename
512 <<
" has size " << testSize << std::endl;
518 DiffType::Pointer diff = DiffType::New();
519 diff->SetValidInput( baselineReader->GetOutput() );
520 diff->SetTestInput( testReader->GetOutput() );
521 diff->SetDifferenceThreshold(intensityTolerance);
522 diff->SetToleranceRadius(radiusTolerance);
523 diff->UpdateLargestPossibleRegion();
526 status = diff->GetNumberOfPixelsWithDifferences();
528 if ( ! reportErrors )
532 std::string shortFilename = itksys::SystemTools::GetFilenameName( baselineImageFilename );
534 std::cout <<
"<DartMeasurement name=\"ImageError " << shortFilename
535 <<
"\" type=\"numeric/double\">";
537 std::cout <<
"</DartMeasurement>" << std::endl;
541 if ( ( status > numberOfPixelsTolerance ) && reportErrors )
545 std::cout <<
"<DartMeasurement name=\"ImageError\" type=\"numeric/double\">";
547 std::cout <<
"</DartMeasurement>" << std::endl;
551 std::cout <<
"<DartMeasurement name=\"ImageError Minimum\" type=\"numeric/double\">";
552 std::cout << diff->GetMinimumDifference() <<
"</DartMeasurement>" << std::endl;
554 std::cout <<
"<DartMeasurement name=\"ImageError Maximum\" type=\"numeric/double\">";
555 std::cout << diff->GetMaximumDifference() <<
"</DartMeasurement>" << std::endl;
557 std::cout <<
"<DartMeasurement name=\"ImageError Mean\" type=\"numeric/double\">";
558 std::cout << diff->GetMeanDifference() <<
"</DartMeasurement>" << std::endl;
565 OutputType::SizeType size; size.Fill(0);
567 RescaleType::Pointer rescale = RescaleType::New();
570 rescale->SetInput( diff->GetOutput() );
571 rescale->UpdateLargestPossibleRegion();
572 size = rescale->GetOutput()->GetLargestPossibleRegion().GetSize();
576 OutputType::IndexType index; index.Fill(0);
579 index[i] = size[i] / 2;
585 region.SetIndex(index);
587 region.SetSize(size);
589 ExtractType::Pointer extract = ExtractType::New();
590 extract->SetDirectionCollapseToIdentity();
591 extract->SetInput( rescale->GetOutput() );
592 extract->SetExtractionRegion(region);
594 WriterType::Pointer writer = WriterType::New();
595 writer->SetInput( extract->GetOutput() );
597 std::ostringstream diffName;
598 diffName << testImageFilename <<
".diff.png";
601 rescale->SetInput( diff->GetOutput() );
604 catch (
const std::exception & e )
606 std::cerr <<
"Error during rescale of " << diffName.str() << std::endl;
607 std::cerr << e.what() <<
"\n";
611 std::cerr <<
"Error during rescale of " << diffName.str() << std::endl;
613 writer->SetFileName( diffName.str().c_str() );
618 catch (
const std::exception & e )
620 std::cerr <<
"Error during write of " << diffName.str() << std::endl;
621 std::cerr << e.what() <<
"\n";
625 std::cerr <<
"Error during write of " << diffName.str() << std::endl;
628 std::cout <<
"<DartMeasurementFile name=\"DifferenceImage\" type=\"image/png\">";
629 std::cout << diffName.str();
630 std::cout <<
"</DartMeasurementFile>" << std::endl;
632 std::ostringstream baseName;
633 baseName << testImageFilename <<
".base.png";
636 rescale->SetInput( baselineReader->GetOutput() );
639 catch (
const std::exception & e )
641 std::cerr <<
"Error during rescale of " << baseName.str() << std::endl;
642 std::cerr << e.what() <<
"\n";
646 std::cerr <<
"Error during rescale of " << baseName.str() << std::endl;
650 writer->SetFileName( baseName.str().c_str() );
653 catch (
const std::exception & e )
655 std::cerr <<
"Error during write of " << baseName.str() << std::endl;
656 std::cerr << e.what() <<
"\n";
660 std::cerr <<
"Error during write of " << baseName.str() << std::endl;
663 std::cout <<
"<DartMeasurementFile name=\"BaselineImage\" type=\"image/png\">";
664 std::cout << baseName.str();
665 std::cout <<
"</DartMeasurementFile>" << std::endl;
667 std::ostringstream testName;
668 testName << testImageFilename <<
".test.png";
671 rescale->SetInput( testReader->GetOutput() );
674 catch (
const std::exception & e )
676 std::cerr <<
"Error during rescale of " << testName.str() << std::endl;
677 std::cerr << e.what() <<
"\n";
681 std::cerr <<
"Error during rescale of " << testName.str() << std::endl;
685 writer->SetFileName( testName.str().c_str() );
688 catch (
const std::exception & e )
690 std::cerr <<
"Error during write of " << testName.str() << std::endl;
691 std::cerr << e.what() <<
"\n";
695 std::cerr <<
"Error during write of " << testName.str() << std::endl;
698 std::cout <<
"<DartMeasurementFile name=\"TestImage\" type=\"image/png\">";
699 std::cout << testName.str();
700 std::cout <<
"</DartMeasurementFile>" << std::endl;
702 return ( status > numberOfPixelsTolerance ) ?
static_cast<int>(status) : 0;
705 template<
typename TImageType >
708 typedef TImageType ImageType;
713 typename ReaderType::Pointer testReader = ReaderType::New();
714 testReader->SetFileName(testImageFilename);
717 testReader->UpdateLargestPossibleRegion();
721 std::cerr <<
"Exception detected while reading " << testImageFilename <<
" : " << e.
GetDescription() << std::endl;
727 typename HashFilterType::Pointer hasher = HashFilterType::New();
728 hasher->SetInput( testReader->GetOutput() );
731 return hasher->GetHash();
736 const std::vector<std::string> &baselineMD5Vector )
743 itkGenericExceptionMacro(
"Unable to determine ImageIO reader for \"" << testImageFilename <<
"\"" );
747 iobase->SetFileName( testImageFilename );
748 iobase->ReadImageInformation();
753 std::string testMD5 =
"";
754 switch(componentType)
757 testMD5 = ComputeHash< itk::VectorImage<char, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
760 testMD5 = ComputeHash< itk::VectorImage<unsigned char, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
763 testMD5 = ComputeHash< itk::VectorImage<short, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
766 testMD5 = ComputeHash< itk::VectorImage<unsigned short, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
769 testMD5 = ComputeHash< itk::VectorImage<int, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
772 testMD5 = ComputeHash< itk::VectorImage<unsigned int, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
775 testMD5 = ComputeHash< itk::VectorImage<long, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
778 testMD5 = ComputeHash< itk::VectorImage<unsigned long, ITK_TEST_DIMENSION_MAX> >( testImageFilename );
782 std::cerr <<
"Hashing is not supporting for float and double images." << std::endl;
783 itkGenericExceptionMacro(
"Hashing is not supported for images of float or doubles." );
788 itkGenericExceptionMacro(
"Logic error!" );
791 std::vector<std::string>::const_iterator iter = baselineMD5Vector.begin();
792 assert( baselineMD5Vector.size() );
795 if ( *iter == testMD5 )
801 while (++iter != baselineMD5Vector.end() );
804 std::cout <<
"<DartMeasurement name=\"TestMD5\" type=\"text/string\">";
805 std::cout << testMD5;
806 std::cout <<
"</DartMeasurement>" << std::endl;
810 for ( iter = baselineMD5Vector.begin(); iter != baselineMD5Vector.end(); ++iter )
812 std::cout <<
"<DartMeasurement name=\"BaselineMD5\" type=\"text/string\">";
814 std::cout <<
"</DartMeasurement>" << std::endl;
826 ReaderType::Pointer reader = ReaderType::New();
827 reader->SetFileName( testImageFilename );
828 reader->UpdateLargestPossibleRegion();
830 ImageType::SizeType size;
831 size = reader->GetOutput()->GetLargestPossibleRegion().GetSize();
835 ImageType::IndexType index; index.Fill(0);
838 index[i] = size[i] / 2;
844 ImageType::RegionType region;
845 region.SetIndex(index);
847 region.SetSize(size);
849 ExtractType::Pointer extract = ExtractType::New();
850 extract->SetDirectionCollapseToIdentity();
851 extract->SetInput( reader->GetOutput() );
852 extract->SetExtractionRegion(region);
854 RescaleType::Pointer rescale = RescaleType::New();
857 rescale->SetInput( extract->GetOutput() );
859 WriterType::Pointer writer = WriterType::New();
860 writer->SetInput( rescale->GetOutput() );
863 std::ostringstream testName;
864 testName << testImageFilename <<
".test.png";
866 writer->SetFileName( testName.str().c_str() );
870 rescale->UpdateLargestPossibleRegion();
873 catch (
const std::exception &
e )
875 std::cerr <<
"Error during rescale and writing of " << testName.str()<< std::endl;
876 std::cerr << e.what() <<
"\n";
880 std::cerr <<
"Unknow error during rescale and writing of " << testName.str() << std::endl;
883 std::cout <<
"<DartMeasurementFile name=\"TestImage\" type=\"image/png\">";
884 std::cout << testName.str();
885 std::cout <<
"</DartMeasurementFile>" << std::endl;
901 std::map< std::string, int > baselines;
902 baselines[std::string(baselineFilename)] = 0;
904 std::string originalBaseline(baselineFilename);
907 std::string::size_type suffixPos = originalBaseline.rfind(
".");
909 if ( suffixPos != std::string::npos )
911 suffix = originalBaseline.substr( suffixPos, originalBaseline.length() );
912 originalBaseline.erase( suffixPos, originalBaseline.length() );
916 std::ostringstream filename;
917 filename << originalBaseline <<
"." << x << suffix;
918 std::ifstream filestream( filename.str().c_str() );
923 baselines[filename.str()] = 0;
931 #include "itkTestingComparisonImageFilter.hxx"