1 /* ---------------------------------------------------------------------- 2 * Project: CMSIS DSP Library 3 * Title: Test.h 4 * Description: Test Framework Header 5 * 6 * $Date: 20. June 2019 7 * $Revision: V1.0.0 8 * 9 * Target Processor: Cortex-M cores 10 * -------------------------------------------------------------------- */ 11 /* 12 * Copyright (C) 2010-2019 ARM Limited or its affiliates. All rights reserved. 13 * 14 * SPDX-License-Identifier: Apache-2.0 15 * 16 * Licensed under the Apache License, Version 2.0 (the License); you may 17 * not use this file except in compliance with the License. 18 * You may obtain a copy of the License at 19 * 20 * www.apache.org/licenses/LICENSE-2.0 21 * 22 * Unless required by applicable law or agreed to in writing, software 23 * distributed under the License is distributed on an AS IS BASIS, WITHOUT 24 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 25 * See the License for the specific language governing permissions and 26 * limitations under the License. 27 */ 28 #ifndef _TEST_H_ 29 #define _TEST_H_ 30 31 #include <cstdlib> 32 #include <vector> 33 #include <cstdio> 34 #include "arm_math_types.h" 35 #include "arm_math_types_f16.h" 36 37 38 // This special value means no limit on the number of samples. 39 // It is used when importing patterns and we want to read 40 // all the samples contained in the pattern. 41 #define MAX_NB_SAMPLES 0 42 43 // Pattern files are containing hexadecimal values. 44 // So we need to be able to convert some int into float without convertion 45 #define TOINT16(v) *((uint16_t*)&v) 46 #define TOINT32(v) *((uint32_t*)&v) 47 #define TOINT64(v) *((uint64_t*)&v) 48 // Or convert some float into a uint32 or uint64 without convertion 49 #define TOTYP(TYP,v) *((TYP*)&v) 50 // So it is a cast and not a data conversion. 51 // (uint32_t)1.0 would give 1. We want the hexadecimal representation of the float. 52 // TOINT32(1.0) can be used 53 54 55 namespace Testing 56 { 57 enum TestStatus 58 { 59 kTestFailed=0, 60 kTestPassed=1 61 }; 62 63 /* In Dump only, reference patterns are never read. 64 So tests are failing 65 and we dump output. 66 In this mode we are only interested in the output data and 67 not the test status. 68 69 In test only mode, no output is dumped. 70 */ 71 enum RunningMode 72 { 73 kTestOnly=0, 74 kDumpOnly=1, 75 kTestAndDump=2 76 }; 77 78 79 80 81 // test ID are ID of nodes in the tree of tests. 82 // So a group ID, suite ID or test ID all have the same type 83 // testID_t 84 typedef unsigned long testID_t; 85 typedef unsigned long outputID_t; 86 typedef unsigned long PatternID_t; 87 88 typedef unsigned long testIndex_t; 89 typedef unsigned long nbSamples_t; 90 typedef unsigned long errorID_t; 91 92 93 94 typedef uint32_t cycles_t; 95 typedef unsigned long nbMeasurements_t; 96 97 // parameter value 98 // (always int since we need to be able to iterate on 99 // different parameter values which are often dimensions of 100 // input data) 101 typedef int param_t; 102 // Number of parameters for a given configuration 103 typedef unsigned long nbParameters_t; 104 // Number of parameter configurations 105 typedef unsigned long nbParameterEntries_t; 106 107 // To know if parameter array is malloc buffer or static buffer in C array 108 enum ParameterKind 109 { 110 kStaticBuffer=0, 111 kDynamicBuffer=1, 112 }; 113 114 } 115 116 namespace Client 117 { 118 119 120 121 /* 122 123 Client code 124 125 */ 126 127 128 129 class Suite; 130 class Group; 131 132 133 // Type of a test function 134 // It is not a function pointer (because the function is 135 // a method of a CPP class) 136 typedef void (Suite::*test)(); 137 138 /* 139 140 API of Memory managers used in the test framework 141 142 */ 143 class Memory 144 { 145 public: 146 // Allocate a new buffer of size length 147 // and generate a pointer to this buffer. 148 // It does not imply that any malloc is done. 149 // It depends on how the Memory manager is implemented. 150 virtual char *NewBuffer(size_t length)=0; 151 152 // Free all the memory allocated by the memory manager 153 // and increment the memory generation number. 154 virtual void FreeMemory()=0; 155 156 // Memory allocation errors must be tracked during a test. 157 // The runner should force the test status to FAILED 158 // when a memory error occured. 159 virtual bool HasMemError()=0; 160 161 // When memory manager is supporting tail 162 // then we can check that the tail of the buffer has not been 163 // corrupted. 164 // The tail being the additional words after the end of the buffer allocated 165 // by the memory manager so that there is some seperation between 166 // successive buffers. 167 // When memory manager is not supporting tail, this function should 168 // always succeed. 169 virtual bool IsTailEmpty(char *, size_t)=0; 170 171 172 // Get the memory generation number generation()173 unsigned long generation() 174 { 175 return(m_generation); 176 } 177 178 protected: 179 unsigned long m_generation=0; 180 }; 181 182 183 // A runner is a class driving the tests 184 // It can use information from driving files 185 // or in the future could communicate with a process 186 // on a host computer which would be the real driver of the 187 // testing. 188 // It is following the visitor pattern. IT is the reason for an accept 189 // function in Group class. 190 // Run is the visitor 191 class Runner 192 { 193 public: 194 virtual Testing::TestStatus run(Suite*) = 0; 195 virtual Testing::TestStatus run(Group*) = 0; 196 }; 197 198 // Abstract the IO needs of the test framework. 199 // IO could be done from semihosting, socket, C array in memory etc ... 200 class IO 201 { 202 public: 203 204 /** Read the identification of a node from driver data. 205 206 Update the node kind and node id and local folder. 207 To be used for group and suite. Generally update 208 the path to the folder by using this new local folder 209 which is appended to the path. 210 */ 211 virtual void ReadIdentification()=0; 212 213 /** Read the identification of a node driver data. 214 215 Update the node kind and node id and local folder. 216 */ 217 virtual void ReadTestIdentification()=0; 218 219 /** Read the number of parameters for all the tests in a suite 220 221 Used for benchmarking. Same functions executed with 222 different initializations controlled by the parameters. 223 224 */ 225 virtual Testing::nbParameters_t ReadNbParameters()=0; 226 227 /** Dump the test status 228 229 For format of output, refer to Python script. 230 The format must be coherent with the Python script 231 parsing the output. 232 */ 233 virtual void DispStatus(Testing::TestStatus,Testing::errorID_t,unsigned long,Testing::cycles_t cycles)=0; 234 235 236 /** Dump additional details for the error 237 238 For instance, for SNR error, it may contain the SNR value. 239 */ 240 virtual void DispErrorDetails(const char* )=0; 241 242 /** Dump parameters for a test 243 244 When a test is run several time with different 245 parameters for benchmarking, 246 the parameters are displayed after test status. 247 Line should begin with b 248 */ 249 virtual void DumpParams(std::vector<Testing::param_t>&)=0; 250 251 252 /** Dump an end of group/suite to output 253 254 Used by Python script parsing the output. 255 256 */ 257 virtual void EndGroup()=0; 258 259 /** Get the zize of a pattern in this suite. 260 261 Pattern is identified with an ID. 262 Using the local path and ID, the IO implementatiom should 263 be able to access the pattern. 264 265 The path do not have to be a file path. Just a way 266 to identify patterns in a suite and know 267 how to access them. 268 269 */ 270 virtual Testing::nbSamples_t GetPatternSize(Testing::PatternID_t)=0; 271 272 /** Get the size of a parameter pattern in this suite. 273 274 Parameter is identified with an ID. 275 Using the local path and ID, the IO implementatiom should 276 be able to access the data. 277 278 The path do not have to be a file path. Just a way 279 to identify data in a suite and know 280 how to access them. 281 282 */ 283 //virtual Testing::nbSamples_t GetParameterSize(Testing::PatternID_t id)=0; 284 285 /** Check if some parameters are controlling this test 286 */ 287 virtual bool hasParam()=0; 288 289 /** Get ID of parameter generator 290 */ 291 virtual Testing::PatternID_t getParamID()=0; 292 293 /** Import pattern. 294 295 The nb field can be used to limit the number of samples read 296 to a smaller value than the number of samples available in the 297 pattern. 298 299 */ 300 virtual void ImportPattern_f64(Testing::PatternID_t,char*,Testing::nbSamples_t nb=MAX_NB_SAMPLES)=0; 301 virtual void ImportPattern_f32(Testing::PatternID_t,char*,Testing::nbSamples_t nb=MAX_NB_SAMPLES)=0; 302 #if !defined( __CC_ARM ) && defined(ARM_FLOAT16_SUPPORTED) 303 virtual void ImportPattern_f16(Testing::PatternID_t,char*,Testing::nbSamples_t nb=MAX_NB_SAMPLES)=0; 304 #endif 305 virtual void ImportPattern_q63(Testing::PatternID_t,char*,Testing::nbSamples_t nb=MAX_NB_SAMPLES)=0; 306 virtual void ImportPattern_q31(Testing::PatternID_t,char*,Testing::nbSamples_t nb=MAX_NB_SAMPLES)=0; 307 virtual void ImportPattern_q15(Testing::PatternID_t,char*,Testing::nbSamples_t nb=MAX_NB_SAMPLES)=0; 308 virtual void ImportPattern_q7(Testing::PatternID_t,char*,Testing::nbSamples_t nb=MAX_NB_SAMPLES)=0; 309 virtual void ImportPattern_u64(Testing::PatternID_t,char*,Testing::nbSamples_t nb=MAX_NB_SAMPLES)=0; 310 virtual void ImportPattern_u32(Testing::PatternID_t,char*,Testing::nbSamples_t nb=MAX_NB_SAMPLES)=0; 311 virtual void ImportPattern_u16(Testing::PatternID_t,char*,Testing::nbSamples_t nb=MAX_NB_SAMPLES)=0; 312 virtual void ImportPattern_u8(Testing::PatternID_t,char*,Testing::nbSamples_t nb=MAX_NB_SAMPLES)=0; 313 314 /** Import params. 315 316 This is allocating memory. 317 The runner should free it after use. 318 319 It is not using the Memory manager since tests don't have access 320 to the array of parameters. 321 322 They receive parameters as a vector argument for the setUp fucntion. 323 */ 324 virtual Testing::param_t* ImportParams(Testing::PatternID_t,Testing::nbParameterEntries_t &,Testing::ParameterKind &)=0; 325 326 /** Dump pattern. 327 328 The output ID (and test ID) must be used to uniquely identify 329 the dump. 330 331 332 */ 333 virtual void DumpPattern_f64(Testing::outputID_t,Testing::nbSamples_t nb, float64_t*)=0; 334 virtual void DumpPattern_f32(Testing::outputID_t,Testing::nbSamples_t nb, float32_t*)=0; 335 #if !defined( __CC_ARM ) && defined(ARM_FLOAT16_SUPPORTED) 336 virtual void DumpPattern_f16(Testing::outputID_t,Testing::nbSamples_t nb, float16_t*)=0; 337 #endif 338 virtual void DumpPattern_q63(Testing::outputID_t,Testing::nbSamples_t nb, q63_t*)=0; 339 virtual void DumpPattern_q31(Testing::outputID_t,Testing::nbSamples_t nb, q31_t*)=0; 340 virtual void DumpPattern_q15(Testing::outputID_t,Testing::nbSamples_t nb, q15_t*)=0; 341 virtual void DumpPattern_q7(Testing::outputID_t,Testing::nbSamples_t nb, q7_t*)=0; 342 virtual void DumpPattern_u64(Testing::outputID_t,Testing::nbSamples_t nb, uint64_t*)=0; 343 virtual void DumpPattern_u32(Testing::outputID_t,Testing::nbSamples_t nb, uint32_t*)=0; 344 virtual void DumpPattern_u16(Testing::outputID_t,Testing::nbSamples_t nb, uint16_t*)=0; 345 virtual void DumpPattern_u8(Testing::outputID_t,Testing::nbSamples_t nb, uint8_t*)=0; 346 347 /** Import list of patterns from the driver 348 for current suite. 349 350 This list is used to identify a pattern from its pattern ID. 351 The information of this list (local to the suite) is 352 combined with the path to identify patterns in other part of the class. 353 354 */ 355 virtual void ReadPatternList()=0; 356 357 /** Import list of output from the driver 358 for current suite. 359 360 This list is used to identify an output from its pattern ID (and current test ID) 361 The information of this list (local to the suite) is 362 combined with the path and current test ID 363 to identify output in other part of the class. 364 365 */ 366 virtual void ReadOutputList()=0; 367 368 /** Import list of parameters from the driver 369 for current suite. 370 371 This list is used to control a functions with different parameters 372 for benchmarking purpose. 373 374 A parameter can be a file of parameters or a generator 375 of parameters (cartesian product of lists only). 376 377 */ 378 virtual void ReadParameterList(Testing::nbParameters_t)=0; 379 380 /** Get current node ID 381 group, suite or test. A group of test is considered as a test hence 382 the name of the function. 383 384 */ 385 virtual Testing::testID_t CurrentTestID()=0; 386 }; 387 388 389 // A pattern manager is making the link between 390 // IO and the Memory manager. 391 // It knows how to import patterns into memory or dump 392 // memory into outputs (output which may be different from a file) 393 // The running mode is controlling if dumping is disabled or not. 394 // But cna also be used by the runner to know if test results must be ignored or not. 395 // Pattern manager is used by the tests 396 // In current version load and dump functions are visible to any body. 397 // In theory they should only be visible to Patterns 398 class PatternMgr 399 { 400 public: 401 PatternMgr(IO*,Memory*); 402 403 /** In those loading APIs, nb samples is coming from the pattern read. 404 maxSamples is coming from the test. 405 406 A test does not know what is the length of a pattern since the test 407 has no visiblity on how the pattern is imported (file, serial port, include files 408 etc ...). 409 410 So the test is specifying the max sampls it needs. 411 The pattern is specifying its lengths. 412 413 Those functions are importing at most what is needed and what is available. 414 415 */ 416 float64_t *load_f64(Testing::PatternID_t,Testing::nbSamples_t&,Testing::nbSamples_t maxSamples=MAX_NB_SAMPLES); 417 float32_t *load_f32(Testing::PatternID_t,Testing::nbSamples_t&,Testing::nbSamples_t maxSamples=MAX_NB_SAMPLES); 418 #if !defined( __CC_ARM ) && defined(ARM_FLOAT16_SUPPORTED) 419 float16_t *load_f16(Testing::PatternID_t,Testing::nbSamples_t&,Testing::nbSamples_t maxSamples=MAX_NB_SAMPLES); 420 #endif 421 q63_t *load_q63(Testing::PatternID_t,Testing::nbSamples_t&,Testing::nbSamples_t maxSamples=MAX_NB_SAMPLES); 422 q31_t *load_q31(Testing::PatternID_t,Testing::nbSamples_t&,Testing::nbSamples_t maxSamples=MAX_NB_SAMPLES); 423 q15_t *load_q15(Testing::PatternID_t,Testing::nbSamples_t&,Testing::nbSamples_t maxSamples=MAX_NB_SAMPLES); 424 q7_t *load_q7(Testing::PatternID_t,Testing::nbSamples_t&,Testing::nbSamples_t maxSamples=MAX_NB_SAMPLES); 425 426 uint64_t *load_u64(Testing::PatternID_t,Testing::nbSamples_t&,Testing::nbSamples_t maxSamples=MAX_NB_SAMPLES); 427 uint32_t *load_u32(Testing::PatternID_t,Testing::nbSamples_t&,Testing::nbSamples_t maxSamples=MAX_NB_SAMPLES); 428 uint16_t *load_u16(Testing::PatternID_t,Testing::nbSamples_t&,Testing::nbSamples_t maxSamples=MAX_NB_SAMPLES); 429 uint8_t *load_u8(Testing::PatternID_t,Testing::nbSamples_t&,Testing::nbSamples_t maxSamples=MAX_NB_SAMPLES); 430 431 /** Create a local pattern. 432 433 Generally it is used as output of a test and has no 434 correspondance to a pattern in the suite. 435 436 */ 437 float64_t *local_f64(Testing::nbSamples_t); 438 float32_t *local_f32(Testing::nbSamples_t); 439 #if !defined( __CC_ARM ) && defined(ARM_FLOAT16_SUPPORTED) 440 float16_t *local_f16(Testing::nbSamples_t); 441 #endif 442 q63_t *local_q63(Testing::nbSamples_t); 443 q31_t *local_q31(Testing::nbSamples_t); 444 q15_t *local_q15(Testing::nbSamples_t); 445 q7_t *local_q7(Testing::nbSamples_t); 446 447 uint64_t *local_u64(Testing::nbSamples_t); 448 uint32_t *local_u32(Testing::nbSamples_t); 449 uint16_t *local_u16(Testing::nbSamples_t); 450 uint8_t *local_u8(Testing::nbSamples_t); 451 452 /** Dump a pattern 453 454 */ 455 void dumpPattern_f64(Testing::outputID_t,Testing::nbSamples_t,float64_t*); 456 void dumpPattern_f32(Testing::outputID_t,Testing::nbSamples_t,float32_t*); 457 #if !defined( __CC_ARM ) && defined(ARM_FLOAT16_SUPPORTED) 458 void dumpPattern_f16(Testing::outputID_t,Testing::nbSamples_t,float16_t*); 459 #endif 460 461 void dumpPattern_q63(Testing::outputID_t,Testing::nbSamples_t,q63_t*); 462 void dumpPattern_q31(Testing::outputID_t,Testing::nbSamples_t,q31_t*); 463 void dumpPattern_q15(Testing::outputID_t,Testing::nbSamples_t,q15_t*); 464 void dumpPattern_q7(Testing::outputID_t,Testing::nbSamples_t,q7_t*); 465 466 void dumpPattern_u64(Testing::outputID_t,Testing::nbSamples_t,uint64_t*); 467 void dumpPattern_u32(Testing::outputID_t,Testing::nbSamples_t,uint32_t*); 468 void dumpPattern_u16(Testing::outputID_t,Testing::nbSamples_t,uint16_t*); 469 void dumpPattern_u8(Testing::outputID_t,Testing::nbSamples_t,uint8_t*); 470 471 /** Free all allocated patterns. 472 473 Just wrapper around the memory manager free function. 474 475 */ 476 void freeAll(); 477 478 /** MeMory manager generation 479 480 */ generation()481 unsigned long generation() 482 { 483 return(m_mem->generation()); 484 } 485 486 // Memory allocation errors must be tracked during a test. 487 // The runner should force the test status to FAILED 488 // when a memory error occured. HasMemError()489 bool HasMemError() 490 { 491 return(m_mem->HasMemError()); 492 } 493 494 // Set by the runner when in dump mode setDumpMode()495 void setDumpMode() 496 { 497 this->m_runningMode = Testing::kDumpOnly; 498 } 499 setTestAndDumpMode()500 void setTestAndDumpMode() 501 { 502 this->m_runningMode = Testing::kTestAndDump; 503 } 504 runningMode()505 Testing::RunningMode runningMode() 506 { 507 return(this->m_runningMode); 508 } 509 IsTailEmpty(char * ptr,size_t length)510 bool IsTailEmpty(char *ptr, size_t length) 511 { 512 return(m_mem->IsTailEmpty(ptr,length)); 513 } 514 515 private: 516 IO *m_io; 517 Memory *m_mem; 518 Testing::RunningMode m_runningMode=Testing::kTestOnly; 519 520 }; 521 522 // TestContainer which is a node of the tree of tests 523 class TestContainer 524 { 525 public: 526 TestContainer(Testing::testID_t); 527 // Used for implementing the visitor pattern. 528 // The visitor pattern is allowing to implement 529 // different Runner for a tree of tests. 530 virtual Testing::TestStatus accept(Runner* v) = 0; 531 protected: 532 // Node ID (test ID) 533 Testing::testID_t m_containerID; 534 }; 535 536 // A suite object 537 // It contains a list of tests 538 // Methods to get a test from the test ID 539 // Initialization and cleanup (setUp and tearDown) to be called 540 // between each test. 541 // Those functions are used by the Runner to execute the tests 542 class Suite:public TestContainer 543 { 544 public: 545 Suite(Testing::testID_t); 546 547 // Prepare memory for a test 548 // (Load input and reference patterns) 549 virtual void setUp(Testing::testID_t,std::vector<Testing::param_t>&,PatternMgr *mgr)=0; 550 551 // Clean memory after a test 552 // Free all memory 553 // DUmp outputs 554 virtual void tearDown(Testing::testID_t,PatternMgr *mgr)=0; 555 556 // Add a test to be run. 557 void addTest(Testing::testID_t,test aTest); 558 559 // Get a test from its index. Used by runner when iterating 560 // on all the tests. Index is not the test ID. 561 // It is the index in internal list of tests 562 test getTest(Testing::testIndex_t); 563 564 // Get number of test in this suite. 565 // The suite is only containing the active tests 566 // (deprecated tests are never generated by python scripts) 567 int getNbTests(); 568 569 // Used for implementing the visitor pattern. 570 // The visitor pattern is allowing to implement 571 // different Runner for a tree of tests. accept(Runner * v)572 virtual Testing::TestStatus accept(Runner* v) override 573 { 574 return(v->run(this)); 575 } 576 577 // Check if, for benchmark, we want to run the code once 578 // before benchmarking it, to force it to be in the I-cache. 579 bool isForcedInCache(); 580 581 // Change the status of the forceInCache mode. 582 void setForceInCache(bool); 583 584 private: 585 bool m_forcedInCache=false; 586 // List of tests 587 std::vector<test> m_tests; 588 // List of tests IDs (since they are not contiguous 589 // due to deprecation feature in python scripts) 590 std::vector<Testing::testID_t> m_testIds; 591 }; 592 593 594 595 // A group 596 // It is possible to add subgroups to a group 597 // and get a subgroup from its ID. 598 class Group:public TestContainer 599 { 600 public: 601 Group(Testing::testID_t); 602 603 // Add a group or suite to this group. 604 void addContainer(TestContainer*); 605 606 // Get a container from its index. (index is not the node ID) 607 TestContainer *getContainer(Testing::testIndex_t); 608 609 // Get number of containers 610 int getNbContainer(); 611 accept(Runner * v)612 virtual Testing::TestStatus accept(Runner* v) override 613 { 614 return(v->run(this)); 615 } 616 617 public: 618 std::vector<TestContainer*> m_groups; 619 620 }; 621 622 623 624 } 625 626 627 #endif 628
