1#!/usr/bin/env python 2from __future__ import division, print_function 3 4import csv 5import io 6import os 7import struct 8import subprocess 9import sys 10import tempfile 11import unittest 12 13from test_utils import Py23TestCase 14 15try: 16 import gen_esp32part 17except ImportError: 18 sys.path.append('..') 19 import gen_esp32part 20 21SIMPLE_CSV = """ 22# Name,Type,SubType,Offset,Size,Flags 23factory,0,2,65536,1048576, 24""" 25 26LONGER_BINARY_TABLE = b'' 27# type 0x00, subtype 0x00, 28# offset 64KB, size 1MB 29LONGER_BINARY_TABLE += b'\xAA\x50\x00\x00' + \ 30 b'\x00\x00\x01\x00' + \ 31 b'\x00\x00\x10\x00' + \ 32 b'factory\0' + (b'\0' * 8) + \ 33 b'\x00\x00\x00\x00' 34# type 0x01, subtype 0x20, 35# offset 0x110000, size 128KB 36LONGER_BINARY_TABLE += b'\xAA\x50\x01\x20' + \ 37 b'\x00\x00\x11\x00' + \ 38 b'\x00\x02\x00\x00' + \ 39 b'data' + (b'\0' * 12) + \ 40 b'\x00\x00\x00\x00' 41# type 0x10, subtype 0x00, 42# offset 0x150000, size 1MB 43LONGER_BINARY_TABLE += b'\xAA\x50\x10\x00' + \ 44 b'\x00\x00\x15\x00' + \ 45 b'\x00\x10\x00\x00' + \ 46 b'second' + (b'\0' * 10) + \ 47 b'\x00\x00\x00\x00' 48# MD5 checksum 49LONGER_BINARY_TABLE += b'\xEB\xEB' + b'\xFF' * 14 50LONGER_BINARY_TABLE += b'\xf9\xbd\x06\x1b\x45\x68\x6f\x86\x57\x1a\x2c\xd5\x2a\x1d\xa6\x5b' 51# empty partition 52LONGER_BINARY_TABLE += b'\xFF' * 32 53 54 55def _strip_trailing_ffs(binary_table): 56 """ 57 Strip all FFs down to the last 32 bytes (terminating entry) 58 """ 59 while binary_table.endswith(b'\xFF' * 64): 60 binary_table = binary_table[0:len(binary_table) - 32] 61 return binary_table 62 63 64class CSVParserTests(Py23TestCase): 65 66 def test_simple_partition(self): 67 table = gen_esp32part.PartitionTable.from_csv(SIMPLE_CSV) 68 self.assertEqual(len(table), 1) 69 self.assertEqual(table[0].name, 'factory') 70 self.assertEqual(table[0].type, 0) 71 self.assertEqual(table[0].subtype, 2) 72 self.assertEqual(table[0].offset, 65536) 73 self.assertEqual(table[0].size, 1048576) 74 75 def test_require_type(self): 76 csv = """ 77# Name,Type, SubType,Offset,Size 78ihavenotype, 79""" 80 with self.assertRaisesRegex(gen_esp32part.InputError, 'type'): 81 gen_esp32part.PartitionTable.from_csv(csv) 82 83 def test_type_subtype_names(self): 84 csv_magicnumbers = """ 85# Name, Type, SubType, Offset, Size 86myapp, 0, 0,, 0x100000 87myota_0, 0, 0x10,, 0x100000 88myota_1, 0, 0x11,, 0x100000 89myota_15, 0, 0x1f,, 0x100000 90mytest, 0, 0x20,, 0x100000 91myota_status, 1, 0,, 0x2000 92 """ 93 csv_nomagicnumbers = """ 94# Name, Type, SubType, Offset, Size 95myapp, app, factory,, 0x100000 96myota_0, app, ota_0,, 0x100000 97myota_1, app, ota_1,, 0x100000 98myota_15, app, ota_15,, 0x100000 99mytest, app, test,, 0x100000 100myota_status, data, ota,, 0x2000 101""" 102 # make two equivalent partition tables, one using 103 # magic numbers and one using shortcuts. Ensure they match 104 magic = gen_esp32part.PartitionTable.from_csv(csv_magicnumbers) 105 magic.verify() 106 nomagic = gen_esp32part.PartitionTable.from_csv(csv_nomagicnumbers) 107 nomagic.verify() 108 109 self.assertEqual(nomagic['myapp'].type, 0) 110 self.assertEqual(nomagic['myapp'].subtype, 0) 111 self.assertEqual(nomagic['myapp'], magic['myapp']) 112 self.assertEqual(nomagic['myota_0'].type, 0) 113 self.assertEqual(nomagic['myota_0'].subtype, 0x10) 114 self.assertEqual(nomagic['myota_0'], magic['myota_0']) 115 self.assertEqual(nomagic['myota_15'], magic['myota_15']) 116 self.assertEqual(nomagic['mytest'], magic['mytest']) 117 self.assertEqual(nomagic['myota_status'], magic['myota_status']) 118 119 # self.assertEqual(nomagic.to_binary(), magic.to_binary()) 120 121 def test_unit_suffixes(self): 122 csv = """ 123# Name, Type, Subtype, Offset, Size 124one_megabyte, app, factory, 64k, 1M 125""" 126 t = gen_esp32part.PartitionTable.from_csv(csv) 127 t.verify() 128 self.assertEqual(t[0].offset, 64 * 1024) 129 self.assertEqual(t[0].size, 1 * 1024 * 1024) 130 131 def test_default_offsets(self): 132 csv = """ 133# Name, Type, Subtype, Offset, Size 134first, app, factory,, 1M 135second, data, 0x15,, 1M 136minidata, data, 0x40,, 32K 137otherapp, app, factory,, 1M 138 """ 139 t = gen_esp32part.PartitionTable.from_csv(csv) 140 # 'first' 141 self.assertEqual(t[0].offset, 0x010000) # 64KB boundary as it's an app image 142 self.assertEqual(t[0].size, 0x100000) # Size specified in CSV 143 # 'second' 144 self.assertEqual(t[1].offset, 0x110000) # prev offset+size 145 self.assertEqual(t[1].size, 0x100000) # Size specified in CSV 146 # 'minidata' 147 self.assertEqual(t[2].offset, 0x210000) 148 # 'otherapp' 149 self.assertEqual(t[3].offset, 0x220000) # 64KB boundary as it's an app image 150 151 def test_negative_size_to_offset(self): 152 csv = """ 153# Name, Type, Subtype, Offset, Size 154first, app, factory, 0x10000, -2M 155second, data, 0x15, , 1M 156 """ 157 t = gen_esp32part.PartitionTable.from_csv(csv) 158 t.verify() 159 # 'first' 160 self.assertEqual(t[0].offset, 0x10000) # in CSV 161 self.assertEqual(t[0].size, 0x200000 - t[0].offset) # Up to 2M 162 # 'second' 163 self.assertEqual(t[1].offset, 0x200000) # prev offset+size 164 165 def test_overlapping_offsets_fail(self): 166 csv = """ 167first, app, factory, 0x100000, 2M 168second, app, ota_0, 0x200000, 1M 169""" 170 with self.assertRaisesRegex(gen_esp32part.InputError, 'overlap'): 171 t = gen_esp32part.PartitionTable.from_csv(csv) 172 t.verify() 173 174 def test_unique_name_fail(self): 175 csv = """ 176first, app, factory, 0x100000, 1M 177first, app, ota_0, 0x200000, 1M 178""" 179 with self.assertRaisesRegex(gen_esp32part.InputError, 'Partition names must be unique'): 180 t = gen_esp32part.PartitionTable.from_csv(csv) 181 t.verify() 182 183 184class BinaryOutputTests(Py23TestCase): 185 def test_binary_entry(self): 186 csv = """ 187first, 0x30, 0xEE, 0x100400, 0x300000 188""" 189 t = gen_esp32part.PartitionTable.from_csv(csv) 190 tb = _strip_trailing_ffs(t.to_binary()) 191 self.assertEqual(len(tb), 64 + 32) 192 self.assertEqual(b'\xAA\x50', tb[0:2]) # magic 193 self.assertEqual(b'\x30\xee', tb[2:4]) # type, subtype 194 eo, es = struct.unpack('<LL', tb[4:12]) 195 self.assertEqual(eo, 0x100400) # offset 196 self.assertEqual(es, 0x300000) # size 197 self.assertEqual(b'\xEB\xEB' + b'\xFF' * 14, tb[32:48]) 198 self.assertEqual(b'\x43\x03\x3f\x33\x40\x87\x57\x51\x69\x83\x9b\x40\x61\xb1\x27\x26', tb[48:64]) 199 200 def test_multiple_entries(self): 201 csv = """ 202first, 0x30, 0xEE, 0x100400, 0x300000 203second,0x31, 0xEF, , 0x100000 204""" 205 t = gen_esp32part.PartitionTable.from_csv(csv) 206 tb = _strip_trailing_ffs(t.to_binary()) 207 self.assertEqual(len(tb), 96 + 32) 208 self.assertEqual(b'\xAA\x50', tb[0:2]) 209 self.assertEqual(b'\xAA\x50', tb[32:34]) 210 211 def test_encrypted_flag(self): 212 csv = """ 213# Name, Type, Subtype, Offset, Size, Flags 214first, app, factory,, 1M, encrypted 215""" 216 t = gen_esp32part.PartitionTable.from_csv(csv) 217 self.assertTrue(t[0].encrypted) 218 tb = _strip_trailing_ffs(t.to_binary()) 219 tr = gen_esp32part.PartitionTable.from_binary(tb) 220 self.assertTrue(tr[0].encrypted) 221 222 def test_only_empty_subtype_is_not_0(self): 223 csv_txt = """ 224# Name,Type, SubType,Offset,Size 225nvs, data, nvs, , 0x4000, 226otadata, data, ota, , 0x2000, 227phy_init, data, phy, , 0x1000, 228factory, app, factory, , 1M 229ota_0, 0, ota_0, , 1M, 230ota_1, 0, ota_1, , 1M, 231storage, data, , , 512k, 232storage2, data, undefined, , 12k, 233""" 234 t = gen_esp32part.PartitionTable.from_csv(csv_txt) 235 t.verify() 236 self.assertEqual(t[1].name, 'otadata') 237 self.assertEqual(t[1].type, 1) 238 self.assertEqual(t[1].subtype, 0) 239 self.assertEqual(t[6].name, 'storage') 240 self.assertEqual(t[6].type, 1) 241 self.assertEqual(t[6].subtype, 0x06) 242 self.assertEqual(t[7].name, 'storage2') 243 self.assertEqual(t[7].type, 1) 244 self.assertEqual(t[7].subtype, 0x06) 245 246 247class BinaryParserTests(Py23TestCase): 248 def test_parse_one_entry(self): 249 # type 0x30, subtype 0xee, 250 # offset 1MB, size 2MB 251 entry = b'\xAA\x50\x30\xee' + \ 252 b'\x00\x00\x10\x00' + \ 253 b'\x00\x00\x20\x00' + \ 254 b'0123456789abc\0\0\0' + \ 255 b'\x00\x00\x00\x00' + \ 256 b'\xFF' * 32 257 # verify that parsing 32 bytes as a table 258 # or as a single Definition are the same thing 259 t = gen_esp32part.PartitionTable.from_binary(entry) 260 self.assertEqual(len(t), 1) 261 t[0].verify() 262 263 e = gen_esp32part.PartitionDefinition.from_binary(entry[:32]) 264 self.assertEqual(t[0], e) 265 e.verify() 266 267 self.assertEqual(e.type, 0x30) 268 self.assertEqual(e.subtype, 0xEE) 269 self.assertEqual(e.offset, 0x100000) 270 self.assertEqual(e.size, 0x200000) 271 self.assertEqual(e.name, '0123456789abc') 272 273 def test_multiple_entries(self): 274 t = gen_esp32part.PartitionTable.from_binary(LONGER_BINARY_TABLE) 275 t.verify() 276 277 self.assertEqual(3, len(t)) 278 self.assertEqual(t[0].type, gen_esp32part.APP_TYPE) 279 self.assertEqual(t[0].name, 'factory') 280 281 self.assertEqual(t[1].type, gen_esp32part.DATA_TYPE) 282 self.assertEqual(t[1].name, 'data') 283 284 self.assertEqual(t[2].type, 0x10) 285 self.assertEqual(t[2].name, 'second') 286 287 round_trip = _strip_trailing_ffs(t.to_binary()) 288 self.assertEqual(round_trip, LONGER_BINARY_TABLE) 289 290 def test_bad_magic(self): 291 bad_magic = b'OHAI' + \ 292 b'\x00\x00\x10\x00' + \ 293 b'\x00\x00\x20\x00' + \ 294 b'0123456789abc\0\0\0' + \ 295 b'\x00\x00\x00\x00' 296 with self.assertRaisesRegex(gen_esp32part.InputError, 'Invalid magic bytes'): 297 gen_esp32part.PartitionTable.from_binary(bad_magic) 298 299 def test_bad_length(self): 300 bad_length = b'OHAI' + \ 301 b'\x00\x00\x10\x00' + \ 302 b'\x00\x00\x20\x00' + \ 303 b'0123456789' 304 with self.assertRaisesRegex(gen_esp32part.InputError, '32 bytes'): 305 gen_esp32part.PartitionTable.from_binary(bad_length) 306 307 308class CSVOutputTests(Py23TestCase): 309 310 def _readcsv(self, source_str): 311 return list(csv.reader(source_str.split('\n'))) 312 313 def test_output_simple_formatting(self): 314 table = gen_esp32part.PartitionTable.from_csv(SIMPLE_CSV) 315 as_csv = table.to_csv(True) 316 c = self._readcsv(as_csv) 317 # first two lines should start with comments 318 self.assertEqual(c[0][0][0], '#') 319 self.assertEqual(c[1][0][0], '#') 320 row = c[2] 321 self.assertEqual(row[0], 'factory') 322 self.assertEqual(row[1], '0') 323 self.assertEqual(row[2], '2') 324 self.assertEqual(row[3], '0x10000') # reformatted as hex 325 self.assertEqual(row[4], '0x100000') # also hex 326 327 # round trip back to a PartitionTable and check is identical 328 roundtrip = gen_esp32part.PartitionTable.from_csv(as_csv) 329 self.assertEqual(roundtrip, table) 330 331 def test_output_smart_formatting(self): 332 table = gen_esp32part.PartitionTable.from_csv(SIMPLE_CSV) 333 as_csv = table.to_csv(False) 334 c = self._readcsv(as_csv) 335 # first two lines should start with comments 336 self.assertEqual(c[0][0][0], '#') 337 self.assertEqual(c[1][0][0], '#') 338 row = c[2] 339 self.assertEqual(row[0], 'factory') 340 self.assertEqual(row[1], 'app') 341 self.assertEqual(row[2], '2') 342 self.assertEqual(row[3], '0x10000') 343 self.assertEqual(row[4], '1M') 344 345 # round trip back to a PartitionTable and check is identical 346 roundtrip = gen_esp32part.PartitionTable.from_csv(as_csv) 347 self.assertEqual(roundtrip, table) 348 349 350class CommandLineTests(Py23TestCase): 351 352 def test_basic_cmdline(self): 353 try: 354 binpath = tempfile.mktemp() 355 csvpath = tempfile.mktemp() 356 357 # copy binary contents to temp file 358 with open(binpath, 'wb') as f: 359 f.write(LONGER_BINARY_TABLE) 360 361 # run gen_esp32part.py to convert binary file to CSV 362 output = subprocess.check_output([sys.executable, '../gen_esp32part.py', 363 binpath, csvpath], stderr=subprocess.STDOUT) 364 # reopen the CSV and check the generated binary is identical 365 self.assertNotIn(b'WARNING', output) 366 with open(csvpath, 'r') as f: 367 from_csv = gen_esp32part.PartitionTable.from_csv(f.read()) 368 self.assertEqual(_strip_trailing_ffs(from_csv.to_binary()), LONGER_BINARY_TABLE) 369 370 # run gen_esp32part.py to conver the CSV to binary again 371 output = subprocess.check_output([sys.executable, '../gen_esp32part.py', 372 csvpath, binpath], stderr=subprocess.STDOUT) 373 self.assertNotIn(b'WARNING', output) 374 # assert that file reads back as identical 375 with open(binpath, 'rb') as f: 376 binary_readback = f.read() 377 binary_readback = _strip_trailing_ffs(binary_readback) 378 self.assertEqual(binary_readback, LONGER_BINARY_TABLE) 379 380 finally: 381 for path in binpath, csvpath: 382 try: 383 os.remove(path) 384 except OSError: 385 pass 386 387 388class VerificationTests(Py23TestCase): 389 390 def test_bad_alignment(self): 391 csv = """ 392# Name,Type, SubType,Offset,Size 393app,app, factory, 32K, 1M 394""" 395 with self.assertRaisesRegex(gen_esp32part.ValidationError, r'Offset.+not aligned'): 396 t = gen_esp32part.PartitionTable.from_csv(csv) 397 t.verify() 398 399 def test_only_one_otadata(self): 400 csv_txt = """ 401# Name,Type, SubType,Offset,Size 402nvs, data, nvs, , 0x4000, 403otadata, data, ota, , 0x2000, 404otadata2, data, ota, , 0x2000, 405factory, app, factory, , 1M 406ota_0, 0, ota_0, , 1M, 407ota_1, 0, ota_1, , 1M, 408""" 409 with self.assertRaisesRegex(gen_esp32part.InputError, r'Found multiple otadata partitions'): 410 t = gen_esp32part.PartitionTable.from_csv(csv_txt) 411 t.verify() 412 413 def test_otadata_must_have_fixed_size(self): 414 csv_txt = """ 415# Name,Type, SubType,Offset,Size 416nvs, data, nvs, , 0x4000, 417otadata, data, ota, , 0x3000, 418factory, app, factory, , 1M 419ota_0, 0, ota_0, , 1M, 420ota_1, 0, ota_1, , 1M, 421""" 422 with self.assertRaisesRegex(gen_esp32part.InputError, r'otadata partition must have size = 0x2000'): 423 t = gen_esp32part.PartitionTable.from_csv(csv_txt) 424 t.verify() 425 426 def test_app_cannot_have_empty_subtype(self): 427 csv_txt = """ 428# Name,Type, SubType,Offset,Size 429nvs, data, nvs, , 0x4000, 430otadata, data, ota, , 0x2000, 431factory, app, , , 1M 432ota_0, 0, ota_0, , 1M, 433ota_1, 0, ota_1, , 1M, 434""" 435 with self.assertRaisesRegex(gen_esp32part.InputError, r'App partition cannot have an empty subtype'): 436 t = gen_esp32part.PartitionTable.from_csv(csv_txt) 437 t.verify() 438 439 def test_warnings(self): 440 try: 441 sys.stderr = io.StringIO() # capture stderr 442 443 csv_1 = 'app, 1, 2, 32K, 1M\n' 444 gen_esp32part.PartitionTable.from_csv(csv_1).verify() 445 self.assertIn('WARNING', sys.stderr.getvalue()) 446 self.assertIn('partition type', sys.stderr.getvalue()) 447 448 sys.stderr = io.StringIO() 449 csv_2 = 'ota_0, app, ota_1, , 1M\n' 450 gen_esp32part.PartitionTable.from_csv(csv_2).verify() 451 self.assertIn('WARNING', sys.stderr.getvalue()) 452 self.assertIn('partition subtype', sys.stderr.getvalue()) 453 454 sys.stderr = io.StringIO() 455 csv_3 = 'nvs, data, nvs, 0x8800, 32k' 456 gen_esp32part.PartitionTable.from_csv(csv_3).verify() 457 self.assertIn('WARNING', sys.stderr.getvalue()) 458 self.assertIn('not aligned to 0x1000', sys.stderr.getvalue()) 459 460 sys.stderr = io.StringIO() 461 csv_4 = 'factory, app, factory, 0x10000, 0x100100\n' \ 462 'nvs, data, nvs, , 32k' 463 gen_esp32part.PartitionTable.from_csv(csv_4).verify() 464 self.assertIn('WARNING', sys.stderr.getvalue()) 465 self.assertIn('not aligned to 0x1000', sys.stderr.getvalue()) 466 467 finally: 468 sys.stderr = sys.__stderr__ 469 470 471class PartToolTests(Py23TestCase): 472 473 def _run_parttool(self, csvcontents, args): 474 csvpath = tempfile.mktemp() 475 with open(csvpath, 'w') as f: 476 f.write(csvcontents) 477 try: 478 output = subprocess.check_output([sys.executable, '../parttool.py', '-q', '--partition-table-file', 479 csvpath, 'get_partition_info'] + args, 480 stderr=subprocess.STDOUT) 481 self.assertNotIn(b'WARNING', output) 482 return output.strip() 483 finally: 484 os.remove(csvpath) 485 486 def test_find_basic(self): 487 csv = """ 488nvs, data, nvs, 0x9000, 0x4000 489otadata, data, ota, 0xd000, 0x2000 490phy_init, data, phy, 0xf000, 0x1000 491factory, app, factory, 0x10000, 1M 492nvs1_user, data, nvs, 0x110000, 0x4000 493nvs2_user, data, nvs, 0x114000, 0x4000 494nvs_key1, data, nvs_keys, 0x118000, 0x1000, encrypted 495nvs_key2, data, nvs_keys, 0x119000, 0x1000, encrypted 496 """ 497 498 def rpt(args): 499 return self._run_parttool(csv, args) 500 501 self.assertEqual( 502 rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'offset']), b'0x9000') 503 self.assertEqual( 504 rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'size']), b'0x4000') 505 self.assertEqual( 506 rpt(['--partition-name', 'otadata', '--info', 'offset']), b'0xd000') 507 self.assertEqual( 508 rpt(['--partition-boot-default', '--info', 'offset']), b'0x10000') 509 self.assertEqual( 510 rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'name', 'offset', 'size', 'encrypted']), 511 b'nvs 0x9000 0x4000 False') 512 self.assertEqual( 513 rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'name', 'offset', 'size', 'encrypted', '--part_list']), 514 b'nvs 0x9000 0x4000 False nvs1_user 0x110000 0x4000 False nvs2_user 0x114000 0x4000 False') 515 self.assertEqual( 516 rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'name', '--part_list']), 517 b'nvs nvs1_user nvs2_user') 518 self.assertEqual( 519 rpt(['--partition-type', 'data', '--partition-subtype', 'nvs_keys', '--info', 'name', '--part_list']), 520 b'nvs_key1 nvs_key2') 521 self.assertEqual( 522 rpt(['--partition-name', 'nvs', '--info', 'encrypted']), b'False') 523 self.assertEqual( 524 rpt(['--partition-name', 'nvs1_user', '--info', 'encrypted']), b'False') 525 self.assertEqual( 526 rpt(['--partition-name', 'nvs2_user', '--info', 'encrypted']), b'False') 527 self.assertEqual( 528 rpt(['--partition-name', 'nvs_key1', '--info', 'encrypted']), b'True') 529 self.assertEqual( 530 rpt(['--partition-name', 'nvs_key2', '--info', 'encrypted']), b'True') 531 self.assertEqual( 532 rpt(['--partition-type', 'data', '--partition-subtype', 'nvs_keys', '--info', 'name', 'encrypted', '--part_list']), 533 b'nvs_key1 True nvs_key2 True') 534 self.assertEqual( 535 rpt(['--partition-type', 'data', '--partition-subtype', 'nvs', '--info', 'name', 'encrypted', '--part_list']), 536 b'nvs False nvs1_user False nvs2_user False') 537 538 def test_fallback(self): 539 csv = """ 540nvs, data, nvs, 0x9000, 0x4000 541otadata, data, ota, 0xd000, 0x2000 542phy_init, data, phy, 0xf000, 0x1000 543ota_0, app, ota_0, 0x30000, 1M 544ota_1, app, ota_1, , 1M 545 """ 546 547 def rpt(args): 548 return self._run_parttool(csv, args) 549 550 self.assertEqual( 551 rpt(['--partition-type', 'app', '--partition-subtype', 'ota_1', '--info', 'offset']), b'0x130000') 552 self.assertEqual( 553 rpt(['--partition-boot-default', '--info', 'offset']), b'0x30000') # ota_0 554 csv_mod = csv.replace('ota_0', 'ota_2') 555 self.assertEqual( 556 self._run_parttool(csv_mod, ['--partition-boot-default', '--info', 'offset']), 557 b'0x130000') # now default is ota_1 558 559 560if __name__ == '__main__': 561 unittest.main() 562