Blame - scripts/vk_validation_stats.py - platform/external/vulkan-validation-layers

2016-10-31 13:27:36 -0600

[diff] [blame]

136

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

137

def printHelp():

138

print ("Usage:")

139

print (" python vk_validation_stats.py <json_file>")

140

print (" [ -c ]")

141

print (" [ -todo ]")

142

print (" [ -vuid <vuid_name> ]")

143

print (" [ -text [ <text_out_filename>] ]")

144

print (" [ -csv [ <csv_out_filename>] ]")

145

print (" [ -html [ <html_out_filename>] ]")

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

146

print (" [ -export_header ]")

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

147

print (" [ -verbose ]")

148

print (" [ -help ]")

149

print ("\n The vk_validation_stats script parses validation layer source files to")

150

print (" determine the set of valid usage checks and tests currently implemented,")

151

print (" and generates coverage values by comparing against the full set of valid")

152

print (" usage identifiers in the Vulkan-Headers registry file 'validusage.json'")

153

print ("\nArguments: ")

154

print (" <json-file> (required) registry file 'validusage.json'")

155

print (" -c report consistency warnings")

156

print (" -todo report unimplemented VUIDs")

157

print (" -vuid <vuid_name> report status of individual VUID <vuid_name>")

158

print (" -text [filename] output the error database text to <text_database_filename>,")

159

print (" defaults to 'validation_error_database.txt'")

160

print (" -csv [filename] output the error database in csv to <csv_database_filename>,")

161

print (" defaults to 'validation_error_database.csv'")

162

print (" -html [filename] output the error database in html to <html_database_filename>,")

163

print (" defaults to 'validation_error_database.html'")

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

164

print (" -export_header export a new VUID error text header file to <%s>" % header_filename)

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

165

print (" -verbose show your work (to stdout)")

166

167

class ValidationJSON:

168

def __init__(self, filename):

169

self.filename = filename

170

self.explicit_vuids = set()

171

self.implicit_vuids = set()

172

self.all_vuids = set()

173

self.vuid_db = defaultdict(list) # Maps VUID string to list of json-data dicts

174

self.apiversion = ""

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

175

self.duplicate_vuids = set()

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

176

177

# A set of specific regular expression substitutions needed to clean up VUID text

178

self.regex_dict = {}

179

self.regex_dict[re.compile('<.*?>|&(amp;)+lt;|&(amp;)+gt;')] = ""

180

self.regex_dict[re.compile(r'\\\(codeSize \\over 4\\\)')] = "(codeSize/4)"

181

self.regex_dict[re.compile(r'\\\(\\lceil\{\\mathit\{rasterizationSamples} \\over 32}\\rceil\\\)')] = "(rasterizationSamples/32)"

182

# Some fancy punctuation chars that break the Android build...

183

self.regex_dict[re.compile('→')] = "->" # Arrow char

184

self.regex_dict[re.compile('’')] = "'" # Left-slanting apostrophe to apostrophe

185

self.regex_dict[re.compile('&#822(0|1);')] = "'" # L/R-slanting quotes to apostrophe

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

186

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

187

def read(self):

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

188

self.json_dict = {}

189

if os.path.isfile(self.filename):

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

190

json_file = open(self.filename, 'r', encoding='utf-8')

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

191

self.json_dict = json.load(json_file)

192

json_file.close()

193

if len(self.json_dict) == 0:

194

print("Error: Error loading validusage.json file <%s>" % self.filename)

195

sys.exit(-1)

196

try:

197

version = self.json_dict['version info']

198

validation = self.json_dict['validation']

199

self.apiversion = version['api version']

200

except:

201

print("Error: Failure parsing validusage.json object")

202

sys.exit(-1)

203

204

# Parse vuid from json into local databases

205

for apiname in validation.keys():

206

# print("entrypoint:%s"%apiname)

207

apidict = validation[apiname]

208

for ext in apidict.keys():

209

vlist = apidict[ext]

210

for ventry in vlist:

211

vuid_string = ventry['vuid']

212

if (vuid_string[-5:-1].isdecimal()):

213

self.explicit_vuids.add(vuid_string) # explicit end in 5 numeric chars

214

vtype = 'explicit'

215

else:

216

self.implicit_vuids.add(vuid_string) # otherwise, implicit

217

vtype = 'implicit'

218

vuid_text = ventry['text']

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

219

for regex, replacement in self.regex_dict.items():

220

vuid_text = re.sub(regex, replacement, vuid_text) # do regex substitution

221

vuid_text = html.unescape(vuid_text) # anything missed by the regex

222

self.vuid_db[vuid_string].append({'api':apiname, 'ext':ext, 'type':vtype, 'text':vuid_text})

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

223

self.all_vuids = self.explicit_vuids | self.implicit_vuids

224

self.duplicate_vuids = set({v for v in self.vuid_db if len(self.vuid_db[v]) > 1})

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

225

if len(self.duplicate_vuids) > 0:

226

print("Warning: duplicate VUIDs found in validusage.json")

227

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

228

229

class ValidationSource:

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

230

def __init__(self, source_file_list, generated_source_file_list, generated_source_directories):

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

231

self.source_files = source_file_list

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

232

self.generated_source_files = generated_source_file_list

233

self.generated_source_dirs = generated_source_directories

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

234

self.vuid_count_dict = {} # dict of vuid values to the count of how much they're used, and location of where they're used

235

self.duplicated_checks = 0

236

self.explicit_vuids = set()

237

self.implicit_vuids = set()

238

self.unassigned_vuids = set()

239

self.all_vuids = set()

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

240

241

if len(self.generated_source_files) > 0:

242

qualified_paths = []

243

for source in self.generated_source_files:

244

for build_dir in self.generated_source_dirs:

245

filepath = '../%s/layers/%s' % (build_dir, source)

246

if os.path.isfile(filepath):

247

qualified_paths.append(filepath)

John Zulauf

dde04c4

2018-01-16 15:32:45 -0700

[diff] [blame]

248

break

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

249

if len(self.generated_source_files) != len(qualified_paths):

Mark Lobodzinski

2d193ac

2017-06-27 09:38:15 -0600

[diff] [blame]

250

print("Error: Unable to locate one or more of the following source files in the %s directories" % (", ".join(generated_source_directories)))

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

251

print(self.generated_source_files)

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

252

print("Failed to locate one or more codegen files in layer source code - cannot proceed.")

John Zulauf

dde04c4

2018-01-16 15:32:45 -0700

[diff] [blame]

253

exit(1)

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

254

else:

255

self.source_files.extend(qualified_paths)

256

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

257

def parse(self):

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

258

prepend = None

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

259

for sf in self.source_files:

Tobin Ehlis

3d1f2bd

2016-12-22 11:19:15 -0700

[diff] [blame]

260

line_num = 0

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

261

with open(sf) as f:

262

for line in f:

Tobin Ehlis

3d1f2bd

2016-12-22 11:19:15 -0700

[diff] [blame]

263

line_num = line_num + 1

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

264

if True in [line.strip().startswith(comment) for comment in ['//', '/*']]:

265

continue

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

266

# Find vuid strings

267

if prepend != None:

268

line = prepend[:-2] + line.lstrip().lstrip('"') # join lines skipping CR, whitespace and trailing/leading quote char

269

prepend = None

270

if any(prefix in line for prefix in vuid_prefixes):

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

271

line_list = line.split()

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

272

273

# A VUID string that has been broken by clang will start with a vuid prefix and end with -, and will be last in the list

274

broken_vuid = line_list[-1].strip('"')

275

if any(broken_vuid.startswith(prefix) for prefix in vuid_prefixes) and broken_vuid.endswith('-'):

prepend = line

continue

vuid_list = []

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

280

for str in line_list:

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

281

if any(prefix in str for prefix in vuid_prefixes):

282

vuid_list.append(str.strip(',);{}"'))

283

for vuid in vuid_list:

284

if vuid not in self.vuid_count_dict:

285

self.vuid_count_dict[vuid] = {}

286

self.vuid_count_dict[vuid]['count'] = 1

287

self.vuid_count_dict[vuid]['file_line'] = []

288

else:

289

if self.vuid_count_dict[vuid]['count'] == 1: # only count first time duplicated

290

self.duplicated_checks = self.duplicated_checks + 1

291

self.vuid_count_dict[vuid]['count'] = self.vuid_count_dict[vuid]['count'] + 1

292

self.vuid_count_dict[vuid]['file_line'].append('%s,%d' % (sf, line_num))

293

# Sort vuids by type

294

for vuid in self.vuid_count_dict.keys():

295

if (vuid.startswith('VUID-')):

296

if (vuid[-5:-1].isdecimal()):

297

self.explicit_vuids.add(vuid) # explicit end in 5 numeric chars

298

else:

299

self.implicit_vuids.add(vuid)

300

elif (vuid.startswith('UNASSIGNED-')):

301

self.unassigned_vuids.add(vuid)

302

else:

303

print("Unable to categorize VUID: %s" % vuid)

304

print("Confused while parsing VUIDs in layer source code - cannot proceed. (FIXME)")

305

exit(-1)

306

self.all_vuids = self.explicit_vuids | self.implicit_vuids | self.unassigned_vuids

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

307

308

# Class to parse the validation layer test source and store testnames

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

309

class ValidationTests:

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

310

def __init__(self, test_file_list, test_group_name=['VkLayerTest', 'VkPositiveLayerTest', 'VkWsiEnabledLayerTest']):

311

self.test_files = test_file_list

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

312

self.test_trigger_txt_list = []

313

for tg in test_group_name:

314

self.test_trigger_txt_list.append('TEST_F(%s' % tg)

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

315

self.explicit_vuids = set()

316

self.implicit_vuids = set()

317

self.unassigned_vuids = set()

318

self.all_vuids = set()

319

#self.test_to_vuids = {} # Map test name to VUIDs tested

320

self.vuid_to_tests = defaultdict(set) # Map VUIDs to set of test names where implemented

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

321

322

# Parse test files into internal data struct

323

def parse(self):

324

# For each test file, parse test names into set

325

grab_next_line = False # handle testname on separate line than wildcard

Tobin Ehlis

9a68c98

2016-12-29 14:51:17 -0700

[diff] [blame]

326

testname = ''

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

327

prepend = None

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

328

for test_file in self.test_files:

329

with open(test_file) as tf:

330

for line in tf:

331

if True in [line.strip().startswith(comment) for comment in ['//', '/*']]:

332

continue

333

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

334

# if line ends in a broken VUID string, fix that before proceeding

335

if prepend != None:

336

line = prepend[:-2] + line.lstrip().lstrip('"') # join lines skipping CR, whitespace and trailing/leading quote char

337

prepend = None

338

if any(prefix in line for prefix in vuid_prefixes):

339

line_list = line.split()

340

341

# A VUID string that has been broken by clang will start with a vuid prefix and end with -, and will be last in the list

342

broken_vuid = line_list[-1].strip('"')

343

if any(broken_vuid.startswith(prefix) for prefix in vuid_prefixes) and broken_vuid.endswith('-'):

prepend = line

continue

if any(ttt in line for ttt in self.test_trigger_txt_list):

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

348

testname = line.split(',')[-1]

349

testname = testname.strip().strip(' {)')

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

350

if ('' == testname):

351

grab_next_line = True

352

continue

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

353

#self.test_to_vuids[testname] = []

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

354

if grab_next_line: # test name on its own line

355

grab_next_line = False

356

testname = testname.strip().strip(' {)')

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

357

#self.test_to_vuids[testname] = []

358

if any(prefix in line for prefix in vuid_prefixes):

359

line_list = re.split('[\s{}[\]()"]+',line)

Tobin Ehlis

71f38c1

2017-01-12 14:26:56 -0700

[diff] [blame]

360

for sub_str in line_list:

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

361

if any(prefix in sub_str for prefix in vuid_prefixes):

362

vuid_str = sub_str.strip(',);:"')

363

self.vuid_to_tests[vuid_str].add(testname)

364

#self.test_to_vuids[testname].append(vuid_str)

365

if (vuid_str.startswith('VUID-')):

366

if (vuid_str[-5:-1].isdecimal()):

367

self.explicit_vuids.add(vuid_str) # explicit end in 5 numeric chars

368

else:

369

self.implicit_vuids.add(vuid_str)

370

elif (vuid_str.startswith('UNASSIGNED-')):

371

self.unassigned_vuids.add(vuid_str)

372

else:

373

print("Unable to categorize VUID: %s" % vuid_str)

374

print("Confused while parsing VUIDs in test code - cannot proceed. (FIXME)")

375

exit(-1)

376

self.all_vuids = self.explicit_vuids | self.implicit_vuids | self.unassigned_vuids

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

377

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

378

# Class to do consistency checking

379

#

380

class Consistency:

381

def __init__(self, all_json, all_checks, all_tests):

382

self.valid = all_json

383

self.checks = all_checks

384

self.tests = all_tests

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

385

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

386

if (dealias_khr):

387

dk = set()

388

for vuid in self.checks:

389

if vuid in khr_aliases:

390

dk.add(khr_aliases[vuid])

391

else:

392

dk.add(vuid)

393

self.checks = dk

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

394

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

395

dk = set()

396

for vuid in self.tests:

397

if vuid in khr_aliases:

398

dk.add(khr_aliases[vuid])

399

else:

400

dk.add(vuid)

401

self.tests = dk

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

402

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

403

# Report undefined VUIDs in source code

404

def undef_vuids_in_layer_code(self):

405

undef_set = self.checks - self.valid

406

undef_set.discard('VUID-Undefined') # don't report Undefined

407

if ignore_unassigned:

408

unassigned = set({uv for uv in undef_set if uv.startswith('UNASSIGNED-')})

409

undef_set = undef_set - unassigned

410

if (len(undef_set) > 0):

411

print("\nFollowing VUIDs found in layer code are not defined in validusage.json (%d):" % len(undef_set))

412

undef = list(undef_set)

undef.sort()

for vuid in undef:

print(" %s" % vuid)

return False

return True

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

418

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

419

# Report undefined VUIDs in tests

420

def undef_vuids_in_tests(self):

421

undef_set = self.tests - self.valid

422

undef_set.discard('VUID-Undefined') # don't report Undefined

423

if ignore_unassigned:

424

unassigned = set({uv for uv in undef_set if uv.startswith('UNASSIGNED-')})

425

undef_set = undef_set - unassigned

426

if (len(undef_set) > 0):

427

ok = False

428

print("\nFollowing VUIDs found in layer tests are not defined in validusage.json (%d):" % len(undef_set))

429

undef = list(undef_set)

undef.sort()

for vuid in undef:

print(" %s" % vuid)

return False

return True

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

435

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

436

# Report vuids in tests that are not in source

437

def vuids_tested_not_checked(self):

438

undef_set = self.tests - self.checks

439

undef_set.discard('VUID-Undefined') # don't report Undefined

440

if ignore_unassigned:

441

unassigned = set()

442

for vuid in undef_set:

443

if vuid.startswith('UNASSIGNED-'):

444

unassigned.add(vuid)

445

undef_set = undef_set - unassigned

446

if (len(undef_set) > 0):

447

ok = False

448

print("\nFollowing VUIDs found in tests but are not checked in layer code (%d):" % len(undef_set))

449

undef = list(undef_set)

undef.sort()

for vuid in undef:

print(" %s" % vuid)

return False

return True

# TODO: Explicit checked VUIDs which have no test

457

# def explicit_vuids_checked_not_tested(self):

458

459

460

# Class to output database in various flavors

461

#

462

class OutputDatabase:

463

def __init__(self, val_json, val_source, val_tests):

464

self.vj = val_json

465

self.vs = val_source

466

self.vt = val_tests

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

467

self.header_preamble = """/* THIS FILE IS GENERATED. DO NOT EDIT. */

468

/* (scripts/vk_validation_stats.py) */

/*

* Vulkan

*

*

* Licensed under the Apache License, Version 2.0 (the "License");

476

* you may not use this file except in compliance with the License.

477

* You may obtain a copy of the License at

478

*

479

* http://www.apache.org/licenses/LICENSE-2.0

480

*

481

* Unless required by applicable law or agreed to in writing, software

482

* distributed under the License is distributed on an "AS IS" BASIS,

483

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

484

* See the License for the specific language governing permissions and

485

* limitations under the License.

486

*

487

* Author: Tobin Ehlis <tobine@google.com>

488

* Author: Dave Houlton <daveh@lunarg.com>

*/

#pragma once

// Disable auto-formatting for generated file

494

// clang-format off

495

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

496

// Mapping from VUID string to the corresponding spec text

Dave Houlton

4d9b2f8

2018-10-24 18:21:06 -0600

[diff] [blame^]

497

typedef struct _vuid_spec_text_pair {

498

const char * vuid;

499

const char * spec_text;

500

} vuid_spec_text_pair;

501

502

static const vuid_spec_text_pair vuid_spec_text[] = {

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

503

"""

504

self.header_postamble = """};

Dave Houlton

4d9b2f8

2018-10-24 18:21:06 -0600

[diff] [blame^]

505

"""

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

506

self.spec_url = "https://www.khronos.org/registry/vulkan/specs/1.1-extensions/html/vkspec.html"

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

507

508

def dump_txt(self):

509

print("\n Dumping database to text file: %s" % txt_filename)

510

with open (txt_filename, 'w') as txt:

511

txt.write("## VUID Database\n")

512

513

vuid_list = list(self.vj.all_vuids)

514

vuid_list.sort()

515

for vuid in vuid_list:

516

db_list = self.vj.vuid_db[vuid]

517

db_list.sort(key=operator.itemgetter('ext')) # sort list to ease diffs of output file

518

for db_entry in db_list:

519

checked = 'N'

520

if vuid in self.vs.all_vuids:

521

checked = 'Y'

522

test = 'None'

523

if vuid in self.vt.vuid_to_tests:

524

test_list = list(self.vt.vuid_to_tests[vuid])

525

test_list.sort() # sort tests, for diff-ability

526

sep = ', '

527

test = sep.join(test_list)

528

529

txt.write("%s | %s | %s | %s | %s | %s | %s\n" % (vuid, checked, test, db_entry['type'], db_entry['api'], db_entry['ext'], db_entry['text']))

530

531

def dump_csv(self):

532

print("\n Dumping database to csv file: %s" % csv_filename)

533

with open (csv_filename, 'w', newline='') as csvfile:

534

cw = csv.writer(csvfile)

535

cw.writerow(['VUID_NAME','CHECKED','TEST','TYPE','API/STRUCT','EXTENSION','VUID_TEXT'])

536

vuid_list = list(self.vj.all_vuids)

537

vuid_list.sort()

538

for vuid in vuid_list:

539

for db_entry in self.vj.vuid_db[vuid]:

540

row = [vuid]

541

if vuid in self.vs.all_vuids:

row.append('Y')

else:

row.append('N')

test = 'None'

if vuid in self.vt.vuid_to_tests:

547

sep = ', '

548

test = sep.join(self.vt.vuid_to_tests[vuid])

549

row.append(test)

550

row.append(db_entry['type'])

551

row.append(db_entry['api'])

552

row.append(db_entry['ext'])

553

row.append(db_entry['text'])

cw.writerow(row)

def dump_html(self):

print("\n Dumping database to html file: %s" % html_filename)

558

preamble = '<!DOCTYPE html>\n<html>\n<head>\n<style>\ntable, th, td {\n border: 1px solid black;\n border-collapse: collapse; \n}\n</style>\n<body>\n<h2>Valid Usage Database</h2>\n<font size="2" face="Arial">\n<table style="width:100%">\n'

559

headers = '<tr><th>VUID NAME</th><th>CHECKED</th><th>TEST</th><th>TYPE</th><th>API/STRUCT</th><th>EXTENSION</th><th>VUID TEXT</th></tr>\n'

560

with open (html_filename, 'w') as hfile:

561

hfile.write(preamble)

562

hfile.write(headers)

563

vuid_list = list(self.vj.all_vuids)

564

vuid_list.sort()

565

for vuid in vuid_list:

566

for db_entry in self.vj.vuid_db[vuid]:

567

hfile.write('<tr><th>%s</th>' % vuid)

568

checked = '<span style="color:red;">N</span>'

569

if vuid in self.vs.all_vuids:

570

checked = '<span style="color:limegreen;">Y</span>'

571

hfile.write('<th>%s</th>' % checked)

572

test = 'None'

573

if vuid in self.vt.vuid_to_tests:

574

sep = ', '

575

test = sep.join(self.vt.vuid_to_tests[vuid])

576

hfile.write('<th>%s</th>' % test)

577

hfile.write('<th>%s</th>' % db_entry['type'])

578

hfile.write('<th>%s</th>' % db_entry['api'])

579

hfile.write('<th>%s</th>' % db_entry['ext'])

580

hfile.write('<th>%s</th></tr>\n' % db_entry['text'])

581

hfile.write('</table>\n</body>\n</html>\n')

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

582

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

583

def export_header(self):

584

print("\n Exporting header file to: %s" % header_filename)

585

with open (header_filename, 'w') as hfile:

586

hfile.write(self.header_preamble)

587

vuid_list = list(self.vj.all_vuids)

588

vuid_list.sort()

589

for vuid in vuid_list:

590

db_entry = self.vj.vuid_db[vuid][0]

591

hfile.write(' {"%s", "%s (%s#%s)"},\n' % (vuid, db_entry['text'].strip(' '), self.spec_url, vuid))

592

# For multiply-defined VUIDs, include versions with extension appended

593

if len(self.vj.vuid_db[vuid]) > 1:

594

for db_entry in self.vj.vuid_db[vuid]:

595

hfile.write(' {"%s[%s]", "%s (%s#%s)"},\n' % (vuid, db_entry['ext'].strip(' '), db_entry['text'].strip(' '), self.spec_url, vuid))

596

hfile.write(self.header_postamble)

597

Mark Lobodzinski

b44c54a

2017-06-12 12:02:38 -0600

[diff] [blame]

598

def main(argv):

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

global verbose_mode

global txt_filename

global csv_filename

global html_filename

run_consistency = False

605

report_unimplemented = False

get_vuid_status = ''

txt_out = False

csv_out = False

html_out = False

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

610

header_out = False

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

if (1 > len(argv)):

printHelp()

sys.exit()

# Parse script args

json_filename = argv[0]

618

i = 1

619

while (i < len(argv)):

arg = argv[i]

i = i + 1

if (arg == '-c'):

run_consistency = True

624

elif (arg == '-vuid'):

625

get_vuid_status = argv[i]

626

i = i + 1

627

elif (arg == '-todo'):

628

report_unimplemented = True

Karl Schultz

49d66bb

2018-07-09 16:24:46 -0600

[diff] [blame]

629

elif (arg == '-text'):

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

630

txt_out = True

631

# Set filename if supplied, else use default

632

if i < len(argv) and not argv[i].startswith('-'):

633

txt_filename = argv[i]

634

i = i + 1

635

elif (arg == '-csv'):

636

csv_out = True

637

# Set filename if supplied, else use default

638

if i < len(argv) and not argv[i].startswith('-'):

639

csv_filename = argv[i]

640

i = i + 1

641

elif (arg == '-html'):

642

html_out = True

643

# Set filename if supplied, else use default

644

if i < len(argv) and not argv[i].startswith('-'):

645

html_filename = argv[i]

646

i = i + 1

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

647

elif (arg == '-export_header'):

648

header_out = True

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

649

elif (arg in ['-verbose']):

650

verbose_mode = True

651

elif (arg in ['-help', '-h']):

printHelp()

sys.exit()

else:

print("Unrecognized argument: %s\n" % arg)

printHelp()

sys.exit()

Tobin Ehlis

2016-12-05 14:50:03 -0700

[diff] [blame]

659

result = 0 # Non-zero result indicates an error case

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

660

661

# Parse validusage json

662

val_json = ValidationJSON(json_filename)

663

val_json.read()

664

exp_json = len(val_json.explicit_vuids)

665

imp_json = len(val_json.implicit_vuids)

666

all_json = len(val_json.all_vuids)

667

if verbose_mode:

668

print("Found %d unique error vuids in validusage.json file." % all_json)

669

print(" %d explicit" % exp_json)

670

print(" %d implicit" % imp_json)

671

if len(val_json.duplicate_vuids) > 0:

672

print("%d VUIDs appear in validusage.json more than once." % len(val_json.duplicate_vuids))

673

for vuid in val_json.duplicate_vuids:

674

print(" %s" % vuid)

675

for ext in val_json.vuid_db[vuid]:

676

print(" with extension: %s" % ext['ext'])

677

678

# Parse layer source files

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

679

val_source = ValidationSource(layer_source_files, generated_layer_source_files, generated_layer_source_directories)

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

680

val_source.parse()

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

681

exp_checks = len(val_source.explicit_vuids)

682

imp_checks = len(val_source.implicit_vuids)

683

all_checks = len(val_source.vuid_count_dict.keys())

684

if verbose_mode:

685

print("Found %d unique vuid checks in layer source code." % all_checks)

686

print(" %d explicit" % exp_checks)

687

print(" %d implicit" % imp_checks)

688

print(" %d unassigned" % len(val_source.unassigned_vuids))

689

print(" %d checks are implemented more that once" % val_source.duplicated_checks)

690

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

691

# Parse test files

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

692

val_tests = ValidationTests([test_file, ])

693

val_tests.parse()

694

exp_tests = len(val_tests.explicit_vuids)

695

imp_tests = len(val_tests.implicit_vuids)

696

all_tests = len(val_tests.all_vuids)

Mark Lobodzinski

060a8e3

2018-01-08 09:08:06 -0700

[diff] [blame]

697

if verbose_mode:

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

698

print("Found %d unique error vuids in test file %s." % (all_tests, test_file))

699

print(" %d explicit" % exp_tests)

700

print(" %d implicit" % imp_tests)

701

print(" %d unassigned" % len(val_tests.unassigned_vuids))

Mike Weiblen

2017-02-03 12:44:53 -0700

[diff] [blame]

702

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

703

# Process stats

704

print("\nValidation Statistics (using validusage.json version %s)" % val_json.apiversion)

705

print(" VUIDs defined in JSON file: %04d explicit, %04d implicit, %04d total." % (exp_json, imp_json, all_json))

706

print(" VUIDs checked in layer code: %04d explicit, %04d implicit, %04d total." % (exp_checks, imp_checks, all_checks))

707

print(" VUIDs tested in layer tests: %04d explicit, %04d implicit, %04d total." % (exp_tests, imp_tests, all_tests))

708

709

print("\nVUID check coverage")

710

print(" Explicit VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * exp_checks / exp_json), exp_checks, exp_json))

711

print(" Implicit VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * imp_checks / imp_json), imp_checks, imp_json))

712

print(" Overall VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * all_checks / all_json), all_checks, all_json))

Mike Weiblen

2017-02-03 12:44:53 -0700

[diff] [blame]

713

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

714

print("\nVUID test coverage")

715

print(" Explicit VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * exp_tests / exp_checks), exp_tests, exp_checks))

716

print(" Implicit VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * imp_tests / imp_checks), imp_tests, imp_checks))

717

print(" Overall VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * all_tests / all_checks), all_tests, all_checks))

Mike Weiblen

2017-02-03 12:44:53 -0700

[diff] [blame]

718

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

719

# Report status of a single VUID

720

if len(get_vuid_status) > 1:

721

print("\n\nChecking status of <%s>" % get_vuid_status);

722

if get_vuid_status not in val_json.all_vuids:

723

print(' Not a valid VUID string.')

724

else:

725

if get_vuid_status in val_source.explicit_vuids:

726

print(' Implemented!')

727

line_list = val_source.vuid_count_dict[get_vuid_status]['file_line']

728

for line in line_list:

729

print(' => %s' % line)

Dave Houlton

f93bbab

2018-06-26 17:21:12 -0600

[diff] [blame]

730

elif get_vuid_status in val_source.implicit_vuids:

731

print(' Implemented! (Implicit)')

732

line_list = val_source.vuid_count_dict[get_vuid_status]['file_line']

733

for line in line_list:

734

print(' => %s' % line)

Tobin Ehlis

9a68c98

2016-12-29 14:51:17 -0700

[diff] [blame]

735

else:

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

736

print(' Not implemented.')

Dave Houlton

f93bbab

2018-06-26 17:21:12 -0600

[diff] [blame]

737

if get_vuid_status in val_tests.all_vuids:

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

738

print(' Has a test!')

739

test_list = val_tests.vuid_to_tests[get_vuid_status]

740

for test in test_list:

741

print(' => %s' % test)

742

else:

743

print(' Not tested.')

Mike Weiblen

2017-02-03 12:44:53 -0700

[diff] [blame]

744

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

745

# Report unimplemented explicit VUIDs

746

if report_unimplemented:

747

unim_explicit = val_json.explicit_vuids - val_source.explicit_vuids

748

print("\n\n%d explicit VUID checks remain unimplemented:" % len(unim_explicit))

749

ulist = list(unim_explicit)

750

ulist.sort()

751

for vuid in ulist:

752

print(" => %s" % vuid)

# Consistency tests

if run_consistency:

print("\n\nRunning consistency tests...")

757

con = Consistency(val_json.all_vuids, val_source.all_vuids, val_tests.all_vuids)

758

ok = con.undef_vuids_in_layer_code()

759

ok &= con.undef_vuids_in_tests()

760

ok &= con.vuids_tested_not_checked()

761

762

if ok:

763

print(" OK! No inconsistencies found.")

764

765

# Output database in requested format(s)

766

db_out = OutputDatabase(val_json, val_source, val_tests)

if txt_out:

db_out.dump_txt()

if csv_out:

db_out.dump_csv()

if html_out:

db_out.dump_html()

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame]

773

if header_out:

774

db_out.export_header()

Tobin Ehlis

20e3258

2016-12-05 14:50:03 -0700

[diff] [blame]

775

return result

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

776

777

if __name__ == "__main__":

Mark Lobodzinski

b44c54a

2017-06-12 12:02:38 -0600

[diff] [blame]

778

sys.exit(main(sys.argv[1:]))

Tobin Ehlis