Blame - scripts/vk_validation_stats.py - platform/external/vulkan-validation-layers

2016-10-31 13:27:36 -0600

[diff] [blame]

136

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

137

def printHelp():

138

print ("Usage:")

139

print (" python vk_validation_stats.py <json_file>")

140

print (" [ -c ]")

141

print (" [ -todo ]")

142

print (" [ -vuid <vuid_name> ]")

143

print (" [ -text [ <text_out_filename>] ]")

144

print (" [ -csv [ <csv_out_filename>] ]")

145

print (" [ -html [ <html_out_filename>] ]")

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame^]

146

print (" [ -export_header ]")

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

147

print (" [ -verbose ]")

148

print (" [ -help ]")

149

print ("\n The vk_validation_stats script parses validation layer source files to")

150

print (" determine the set of valid usage checks and tests currently implemented,")

151

print (" and generates coverage values by comparing against the full set of valid")

152

print (" usage identifiers in the Vulkan-Headers registry file 'validusage.json'")

153

print ("\nArguments: ")

154

print (" <json-file> (required) registry file 'validusage.json'")

155

print (" -c report consistency warnings")

156

print (" -todo report unimplemented VUIDs")

157

print (" -vuid <vuid_name> report status of individual VUID <vuid_name>")

158

print (" -text [filename] output the error database text to <text_database_filename>,")

159

print (" defaults to 'validation_error_database.txt'")

160

print (" -csv [filename] output the error database in csv to <csv_database_filename>,")

161

print (" defaults to 'validation_error_database.csv'")

162

print (" -html [filename] output the error database in html to <html_database_filename>,")

163

print (" defaults to 'validation_error_database.html'")

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame^]

164

print (" -export_header export a new VUID error text header file to <%s>" % header_filename)

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

165

print (" -verbose show your work (to stdout)")

166

167

class ValidationJSON:

168

def __init__(self, filename):

169

self.filename = filename

170

self.explicit_vuids = set()

171

self.implicit_vuids = set()

172

self.all_vuids = set()

173

self.vuid_db = defaultdict(list) # Maps VUID string to list of json-data dicts

174

self.apiversion = ""

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

175

self.duplicate_vuids = set()

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame^]

176

177

# A set of specific regular expression substitutions needed to clean up VUID text

178

self.regex_dict = {}

179

self.regex_dict[re.compile('<.*?>|&(amp;)+lt;|&(amp;)+gt;')] = ""

180

self.regex_dict[re.compile(r'\\\(codeSize \\over 4\\\)')] = "(codeSize/4)"

181

self.regex_dict[re.compile(r'\\\(\\lceil\{\\mathit\{rasterizationSamples} \\over 32}\\rceil\\\)')] = "(rasterizationSamples/32)"

182

# Some fancy punctuation chars that break the Android build...

183

self.regex_dict[re.compile('→')] = "->" # Arrow char

184

self.regex_dict[re.compile('’')] = "'" # Left-slanting apostrophe to apostrophe

185

self.regex_dict[re.compile('&#822(0|1);')] = "'" # L/R-slanting quotes to apostrophe

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

186

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

187

def read(self):

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

188

self.json_dict = {}

189

if os.path.isfile(self.filename):

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame^]

190

json_file = open(self.filename, 'r', encoding='utf-8')

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

191

self.json_dict = json.load(json_file)

192

json_file.close()

193

if len(self.json_dict) == 0:

194

print("Error: Error loading validusage.json file <%s>" % self.filename)

195

sys.exit(-1)

196

try:

197

version = self.json_dict['version info']

198

validation = self.json_dict['validation']

199

self.apiversion = version['api version']

200

except:

201

print("Error: Failure parsing validusage.json object")

202

sys.exit(-1)

203

204

# Parse vuid from json into local databases

205

for apiname in validation.keys():

206

# print("entrypoint:%s"%apiname)

207

apidict = validation[apiname]

208

for ext in apidict.keys():

209

vlist = apidict[ext]

210

for ventry in vlist:

211

vuid_string = ventry['vuid']

212

if (vuid_string[-5:-1].isdecimal()):

213

self.explicit_vuids.add(vuid_string) # explicit end in 5 numeric chars

214

vtype = 'explicit'

215

else:

216

self.implicit_vuids.add(vuid_string) # otherwise, implicit

217

vtype = 'implicit'

218

vuid_text = ventry['text']

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame^]

219

for regex, replacement in self.regex_dict.items():

220

vuid_text = re.sub(regex, replacement, vuid_text) # do regex substitution

221

vuid_text = html.unescape(vuid_text) # anything missed by the regex

222

self.vuid_db[vuid_string].append({'api':apiname, 'ext':ext, 'type':vtype, 'text':vuid_text})

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

223

self.all_vuids = self.explicit_vuids | self.implicit_vuids

224

self.duplicate_vuids = set({v for v in self.vuid_db if len(self.vuid_db[v]) > 1})

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame^]

225

if len(self.duplicate_vuids) > 0:

226

print("Warning: duplicate VUIDs found in validusage.json")

227

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

228

229

class ValidationSource:

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

230

def __init__(self, source_file_list, generated_source_file_list, generated_source_directories):

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

231

self.source_files = source_file_list

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

232

self.generated_source_files = generated_source_file_list

233

self.generated_source_dirs = generated_source_directories

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

234

self.vuid_count_dict = {} # dict of vuid values to the count of how much they're used, and location of where they're used

235

self.duplicated_checks = 0

236

self.explicit_vuids = set()

237

self.implicit_vuids = set()

238

self.unassigned_vuids = set()

239

self.all_vuids = set()

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

240

241

if len(self.generated_source_files) > 0:

242

qualified_paths = []

243

for source in self.generated_source_files:

244

for build_dir in self.generated_source_dirs:

245

filepath = '../%s/layers/%s' % (build_dir, source)

246

if os.path.isfile(filepath):

247

qualified_paths.append(filepath)

John Zulauf

dde04c4

2018-01-16 15:32:45 -0700

[diff] [blame]

248

break

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

249

if len(self.generated_source_files) != len(qualified_paths):

Mark Lobodzinski

2d193ac

2017-06-27 09:38:15 -0600

[diff] [blame]

250

print("Error: Unable to locate one or more of the following source files in the %s directories" % (", ".join(generated_source_directories)))

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

251

print(self.generated_source_files)

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

252

print("Failed to locate one or more codegen files in layer source code - cannot proceed.")

John Zulauf

dde04c4

2018-01-16 15:32:45 -0700

[diff] [blame]

253

exit(1)

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

254

else:

255

self.source_files.extend(qualified_paths)

256

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

257

def parse(self):

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

258

prepend = None

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

259

for sf in self.source_files:

Tobin Ehlis

3d1f2bd

2016-12-22 11:19:15 -0700

[diff] [blame]

260

line_num = 0

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

261

with open(sf) as f:

262

for line in f:

Tobin Ehlis

3d1f2bd

2016-12-22 11:19:15 -0700

[diff] [blame]

263

line_num = line_num + 1

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

264

if True in [line.strip().startswith(comment) for comment in ['//', '/*']]:

265

continue

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

266

# Find vuid strings

267

if prepend != None:

268

line = prepend[:-2] + line.lstrip().lstrip('"') # join lines skipping CR, whitespace and trailing/leading quote char

269

prepend = None

270

if any(prefix in line for prefix in vuid_prefixes):

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

271

line_list = line.split()

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

272

273

# A VUID string that has been broken by clang will start with a vuid prefix and end with -, and will be last in the list

274

broken_vuid = line_list[-1].strip('"')

275

if any(broken_vuid.startswith(prefix) for prefix in vuid_prefixes) and broken_vuid.endswith('-'):

prepend = line

continue

vuid_list = []

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

280

for str in line_list:

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

281

if any(prefix in str for prefix in vuid_prefixes):

282

vuid_list.append(str.strip(',);{}"'))

283

for vuid in vuid_list:

284

if vuid not in self.vuid_count_dict:

285

self.vuid_count_dict[vuid] = {}

286

self.vuid_count_dict[vuid]['count'] = 1

287

self.vuid_count_dict[vuid]['file_line'] = []

288

else:

289

if self.vuid_count_dict[vuid]['count'] == 1: # only count first time duplicated

290

self.duplicated_checks = self.duplicated_checks + 1

291

self.vuid_count_dict[vuid]['count'] = self.vuid_count_dict[vuid]['count'] + 1

292

self.vuid_count_dict[vuid]['file_line'].append('%s,%d' % (sf, line_num))

293

# Sort vuids by type

294

for vuid in self.vuid_count_dict.keys():

295

if (vuid.startswith('VUID-')):

296

if (vuid[-5:-1].isdecimal()):

297

self.explicit_vuids.add(vuid) # explicit end in 5 numeric chars

298

else:

299

self.implicit_vuids.add(vuid)

300

elif (vuid.startswith('UNASSIGNED-')):

301

self.unassigned_vuids.add(vuid)

302

else:

303

print("Unable to categorize VUID: %s" % vuid)

304

print("Confused while parsing VUIDs in layer source code - cannot proceed. (FIXME)")

305

exit(-1)

306

self.all_vuids = self.explicit_vuids | self.implicit_vuids | self.unassigned_vuids

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

307

308

# Class to parse the validation layer test source and store testnames

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

309

class ValidationTests:

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

310

def __init__(self, test_file_list, test_group_name=['VkLayerTest', 'VkPositiveLayerTest', 'VkWsiEnabledLayerTest']):

311

self.test_files = test_file_list

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

312

self.test_trigger_txt_list = []

313

for tg in test_group_name:

314

self.test_trigger_txt_list.append('TEST_F(%s' % tg)

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

315

self.explicit_vuids = set()

316

self.implicit_vuids = set()

317

self.unassigned_vuids = set()

318

self.all_vuids = set()

319

#self.test_to_vuids = {} # Map test name to VUIDs tested

320

self.vuid_to_tests = defaultdict(set) # Map VUIDs to set of test names where implemented

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

321

322

# Parse test files into internal data struct

323

def parse(self):

324

# For each test file, parse test names into set

325

grab_next_line = False # handle testname on separate line than wildcard

Tobin Ehlis

9a68c98

2016-12-29 14:51:17 -0700

[diff] [blame]

326

testname = ''

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

327

prepend = None

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

328

for test_file in self.test_files:

329

with open(test_file) as tf:

330

for line in tf:

331

if True in [line.strip().startswith(comment) for comment in ['//', '/*']]:

332

continue

333

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

334

# if line ends in a broken VUID string, fix that before proceeding

335

if prepend != None:

336

line = prepend[:-2] + line.lstrip().lstrip('"') # join lines skipping CR, whitespace and trailing/leading quote char

337

prepend = None

338

if any(prefix in line for prefix in vuid_prefixes):

339

line_list = line.split()

340

341

# A VUID string that has been broken by clang will start with a vuid prefix and end with -, and will be last in the list

342

broken_vuid = line_list[-1].strip('"')

343

if any(broken_vuid.startswith(prefix) for prefix in vuid_prefixes) and broken_vuid.endswith('-'):

prepend = line

continue

if any(ttt in line for ttt in self.test_trigger_txt_list):

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

348

testname = line.split(',')[-1]

349

testname = testname.strip().strip(' {)')

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

350

if ('' == testname):

351

grab_next_line = True

352

continue

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

353

#self.test_to_vuids[testname] = []

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

354

if grab_next_line: # test name on its own line

355

grab_next_line = False

356

testname = testname.strip().strip(' {)')

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

357

#self.test_to_vuids[testname] = []

358

if any(prefix in line for prefix in vuid_prefixes):

359

line_list = re.split('[\s{}[\]()"]+',line)

Tobin Ehlis

71f38c1

2017-01-12 14:26:56 -0700

[diff] [blame]

360

for sub_str in line_list:

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

361

if any(prefix in sub_str for prefix in vuid_prefixes):

362

vuid_str = sub_str.strip(',);:"')

363

self.vuid_to_tests[vuid_str].add(testname)

364

#self.test_to_vuids[testname].append(vuid_str)

365

if (vuid_str.startswith('VUID-')):

366

if (vuid_str[-5:-1].isdecimal()):

367

self.explicit_vuids.add(vuid_str) # explicit end in 5 numeric chars

368

else:

369

self.implicit_vuids.add(vuid_str)

370

elif (vuid_str.startswith('UNASSIGNED-')):

371

self.unassigned_vuids.add(vuid_str)

372

else:

373

print("Unable to categorize VUID: %s" % vuid_str)

374

print("Confused while parsing VUIDs in test code - cannot proceed. (FIXME)")

375

exit(-1)

376

self.all_vuids = self.explicit_vuids | self.implicit_vuids | self.unassigned_vuids

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

377

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

378

# Class to do consistency checking

379

#

380

class Consistency:

381

def __init__(self, all_json, all_checks, all_tests):

382

self.valid = all_json

383

self.checks = all_checks

384

self.tests = all_tests

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

385

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

386

if (dealias_khr):

387

dk = set()

388

for vuid in self.checks:

389

if vuid in khr_aliases:

390

dk.add(khr_aliases[vuid])

391

else:

392

dk.add(vuid)

393

self.checks = dk

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

394

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

395

dk = set()

396

for vuid in self.tests:

397

if vuid in khr_aliases:

398

dk.add(khr_aliases[vuid])

399

else:

400

dk.add(vuid)

401

self.tests = dk

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

402

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

403

# Report undefined VUIDs in source code

404

def undef_vuids_in_layer_code(self):

405

undef_set = self.checks - self.valid

406

undef_set.discard('VUID-Undefined') # don't report Undefined

407

if ignore_unassigned:

408

unassigned = set({uv for uv in undef_set if uv.startswith('UNASSIGNED-')})

409

undef_set = undef_set - unassigned

410

if (len(undef_set) > 0):

411

print("\nFollowing VUIDs found in layer code are not defined in validusage.json (%d):" % len(undef_set))

412

undef = list(undef_set)

undef.sort()

for vuid in undef:

print(" %s" % vuid)

return False

return True

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

418

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

419

# Report undefined VUIDs in tests

420

def undef_vuids_in_tests(self):

421

undef_set = self.tests - self.valid

422

undef_set.discard('VUID-Undefined') # don't report Undefined

423

if ignore_unassigned:

424

unassigned = set({uv for uv in undef_set if uv.startswith('UNASSIGNED-')})

425

undef_set = undef_set - unassigned

426

if (len(undef_set) > 0):

427

ok = False

428

print("\nFollowing VUIDs found in layer tests are not defined in validusage.json (%d):" % len(undef_set))

429

undef = list(undef_set)

undef.sort()

for vuid in undef:

print(" %s" % vuid)

return False

return True

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

435

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

436

# Report vuids in tests that are not in source

437

def vuids_tested_not_checked(self):

438

undef_set = self.tests - self.checks

439

undef_set.discard('VUID-Undefined') # don't report Undefined

440

if ignore_unassigned:

441

unassigned = set()

442

for vuid in undef_set:

443

if vuid.startswith('UNASSIGNED-'):

444

unassigned.add(vuid)

445

undef_set = undef_set - unassigned

446

if (len(undef_set) > 0):

447

ok = False

448

print("\nFollowing VUIDs found in tests but are not checked in layer code (%d):" % len(undef_set))

449

undef = list(undef_set)

undef.sort()

for vuid in undef:

print(" %s" % vuid)

return False

return True

# TODO: Explicit checked VUIDs which have no test

457

# def explicit_vuids_checked_not_tested(self):

458

459

460

# Class to output database in various flavors

461

#

462

class OutputDatabase:

463

def __init__(self, val_json, val_source, val_tests):

464

self.vj = val_json

465

self.vs = val_source

466

self.vt = val_tests

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame^]

467

self.header_preamble = """/* THIS FILE IS GENERATED. DO NOT EDIT. */

468

/* (scripts/vk_validation_stats.py) */

/*

* Vulkan

*

*

* Licensed under the Apache License, Version 2.0 (the "License");

476

* you may not use this file except in compliance with the License.

477

* You may obtain a copy of the License at

478

*

479

* http://www.apache.org/licenses/LICENSE-2.0

480

*

481

* Unless required by applicable law or agreed to in writing, software

482

* distributed under the License is distributed on an "AS IS" BASIS,

483

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

484

* See the License for the specific language governing permissions and

485

* limitations under the License.

486

*

487

* Author: Tobin Ehlis <tobine@google.com>

488

* Author: Dave Houlton <daveh@lunarg.com>

*/

#pragma once

// Disable auto-formatting for generated file

// clang-format off

#include <string>

#include <unordered_map>

498

499

// Mapping from VUID string to the corresponding spec text

500

#ifdef VALIDATION_ERROR_MAP_IMPL

501

std::unordered_map<std::string, std::string> vuid_to_error_text_map {

502

"""

503

self.header_postamble = """};

504

#else

505

extern std::unordered_map<std::string, std::string> vuid_to_error_text_map;

506

#endif"""

507

self.spec_url = "https://www.khronos.org/registry/vulkan/specs/1.1-extensions/html/vkspec.html"

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

508

509

def dump_txt(self):

510

print("\n Dumping database to text file: %s" % txt_filename)

511

with open (txt_filename, 'w') as txt:

512

txt.write("## VUID Database\n")

513

514

vuid_list = list(self.vj.all_vuids)

515

vuid_list.sort()

516

for vuid in vuid_list:

517

db_list = self.vj.vuid_db[vuid]

518

db_list.sort(key=operator.itemgetter('ext')) # sort list to ease diffs of output file

519

for db_entry in db_list:

520

checked = 'N'

521

if vuid in self.vs.all_vuids:

522

checked = 'Y'

523

test = 'None'

524

if vuid in self.vt.vuid_to_tests:

525

test_list = list(self.vt.vuid_to_tests[vuid])

526

test_list.sort() # sort tests, for diff-ability

527

sep = ', '

528

test = sep.join(test_list)

529

530

txt.write("%s | %s | %s | %s | %s | %s | %s\n" % (vuid, checked, test, db_entry['type'], db_entry['api'], db_entry['ext'], db_entry['text']))

531

532

def dump_csv(self):

533

print("\n Dumping database to csv file: %s" % csv_filename)

534

with open (csv_filename, 'w', newline='') as csvfile:

535

cw = csv.writer(csvfile)

536

cw.writerow(['VUID_NAME','CHECKED','TEST','TYPE','API/STRUCT','EXTENSION','VUID_TEXT'])

537

vuid_list = list(self.vj.all_vuids)

538

vuid_list.sort()

539

for vuid in vuid_list:

540

for db_entry in self.vj.vuid_db[vuid]:

541

row = [vuid]

542

if vuid in self.vs.all_vuids:

row.append('Y')

else:

row.append('N')

test = 'None'

if vuid in self.vt.vuid_to_tests:

548

sep = ', '

549

test = sep.join(self.vt.vuid_to_tests[vuid])

550

row.append(test)

551

row.append(db_entry['type'])

552

row.append(db_entry['api'])

553

row.append(db_entry['ext'])

554

row.append(db_entry['text'])

cw.writerow(row)

def dump_html(self):

print("\n Dumping database to html file: %s" % html_filename)

559

preamble = '<!DOCTYPE html>\n<html>\n<head>\n<style>\ntable, th, td {\n border: 1px solid black;\n border-collapse: collapse; \n}\n</style>\n<body>\n<h2>Valid Usage Database</h2>\n<font size="2" face="Arial">\n<table style="width:100%">\n'

560

headers = '<tr><th>VUID NAME</th><th>CHECKED</th><th>TEST</th><th>TYPE</th><th>API/STRUCT</th><th>EXTENSION</th><th>VUID TEXT</th></tr>\n'

561

with open (html_filename, 'w') as hfile:

562

hfile.write(preamble)

563

hfile.write(headers)

564

vuid_list = list(self.vj.all_vuids)

565

vuid_list.sort()

566

for vuid in vuid_list:

567

for db_entry in self.vj.vuid_db[vuid]:

568

hfile.write('<tr><th>%s</th>' % vuid)

569

checked = '<span style="color:red;">N</span>'

570

if vuid in self.vs.all_vuids:

571

checked = '<span style="color:limegreen;">Y</span>'

572

hfile.write('<th>%s</th>' % checked)

573

test = 'None'

574

if vuid in self.vt.vuid_to_tests:

575

sep = ', '

576

test = sep.join(self.vt.vuid_to_tests[vuid])

577

hfile.write('<th>%s</th>' % test)

578

hfile.write('<th>%s</th>' % db_entry['type'])

579

hfile.write('<th>%s</th>' % db_entry['api'])

580

hfile.write('<th>%s</th>' % db_entry['ext'])

581

hfile.write('<th>%s</th></tr>\n' % db_entry['text'])

582

hfile.write('</table>\n</body>\n</html>\n')

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

583

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame^]

584

def export_header(self):

585

print("\n Exporting header file to: %s" % header_filename)

586

with open (header_filename, 'w') as hfile:

587

hfile.write(self.header_preamble)

588

vuid_list = list(self.vj.all_vuids)

589

vuid_list.sort()

590

for vuid in vuid_list:

591

db_entry = self.vj.vuid_db[vuid][0]

592

hfile.write(' {"%s", "%s (%s#%s)"},\n' % (vuid, db_entry['text'].strip(' '), self.spec_url, vuid))

593

# For multiply-defined VUIDs, include versions with extension appended

594

if len(self.vj.vuid_db[vuid]) > 1:

595

for db_entry in self.vj.vuid_db[vuid]:

596

hfile.write(' {"%s[%s]", "%s (%s#%s)"},\n' % (vuid, db_entry['ext'].strip(' '), db_entry['text'].strip(' '), self.spec_url, vuid))

597

hfile.write(self.header_postamble)

598

Mark Lobodzinski

b44c54a

2017-06-12 12:02:38 -0600

[diff] [blame]

599

def main(argv):

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

global verbose_mode

global txt_filename

global csv_filename

global html_filename

run_consistency = False

606

report_unimplemented = False

get_vuid_status = ''

txt_out = False

csv_out = False

html_out = False

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame^]

611

header_out = False

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

if (1 > len(argv)):

printHelp()

sys.exit()

# Parse script args

json_filename = argv[0]

619

i = 1

620

while (i < len(argv)):

arg = argv[i]

i = i + 1

if (arg == '-c'):

run_consistency = True

625

elif (arg == '-vuid'):

626

get_vuid_status = argv[i]

627

i = i + 1

628

elif (arg == '-todo'):

629

report_unimplemented = True

Karl Schultz

49d66bb

2018-07-09 16:24:46 -0600

[diff] [blame]

630

elif (arg == '-text'):

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

631

txt_out = True

632

# Set filename if supplied, else use default

633

if i < len(argv) and not argv[i].startswith('-'):

634

txt_filename = argv[i]

635

i = i + 1

636

elif (arg == '-csv'):

637

csv_out = True

638

# Set filename if supplied, else use default

639

if i < len(argv) and not argv[i].startswith('-'):

640

csv_filename = argv[i]

641

i = i + 1

642

elif (arg == '-html'):

643

html_out = True

644

# Set filename if supplied, else use default

645

if i < len(argv) and not argv[i].startswith('-'):

646

html_filename = argv[i]

647

i = i + 1

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame^]

648

elif (arg == '-export_header'):

649

header_out = True

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

650

elif (arg in ['-verbose']):

651

verbose_mode = True

652

elif (arg in ['-help', '-h']):

printHelp()

sys.exit()

else:

print("Unrecognized argument: %s\n" % arg)

printHelp()

sys.exit()

Tobin Ehlis

2016-12-05 14:50:03 -0700

[diff] [blame]

660

result = 0 # Non-zero result indicates an error case

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

661

662

# Parse validusage json

663

val_json = ValidationJSON(json_filename)

664

val_json.read()

665

exp_json = len(val_json.explicit_vuids)

666

imp_json = len(val_json.implicit_vuids)

667

all_json = len(val_json.all_vuids)

668

if verbose_mode:

669

print("Found %d unique error vuids in validusage.json file." % all_json)

670

print(" %d explicit" % exp_json)

671

print(" %d implicit" % imp_json)

672

if len(val_json.duplicate_vuids) > 0:

673

print("%d VUIDs appear in validusage.json more than once." % len(val_json.duplicate_vuids))

674

for vuid in val_json.duplicate_vuids:

675

print(" %s" % vuid)

676

for ext in val_json.vuid_db[vuid]:

677

print(" with extension: %s" % ext['ext'])

678

679

# Parse layer source files

Mark Lobodzinski

2017-06-21 14:44:14 -0600

[diff] [blame]

680

val_source = ValidationSource(layer_source_files, generated_layer_source_files, generated_layer_source_directories)

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

681

val_source.parse()

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

682

exp_checks = len(val_source.explicit_vuids)

683

imp_checks = len(val_source.implicit_vuids)

684

all_checks = len(val_source.vuid_count_dict.keys())

685

if verbose_mode:

686

print("Found %d unique vuid checks in layer source code." % all_checks)

687

print(" %d explicit" % exp_checks)

688

print(" %d implicit" % imp_checks)

689

print(" %d unassigned" % len(val_source.unassigned_vuids))

690

print(" %d checks are implemented more that once" % val_source.duplicated_checks)

691

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

692

# Parse test files

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

693

val_tests = ValidationTests([test_file, ])

694

val_tests.parse()

695

exp_tests = len(val_tests.explicit_vuids)

696

imp_tests = len(val_tests.implicit_vuids)

697

all_tests = len(val_tests.all_vuids)

Mark Lobodzinski

060a8e3

2018-01-08 09:08:06 -0700

[diff] [blame]

698

if verbose_mode:

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

699

print("Found %d unique error vuids in test file %s." % (all_tests, test_file))

700

print(" %d explicit" % exp_tests)

701

print(" %d implicit" % imp_tests)

702

print(" %d unassigned" % len(val_tests.unassigned_vuids))

Mike Weiblen

2017-02-03 12:44:53 -0700

[diff] [blame]

703

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

704

# Process stats

705

print("\nValidation Statistics (using validusage.json version %s)" % val_json.apiversion)

706

print(" VUIDs defined in JSON file: %04d explicit, %04d implicit, %04d total." % (exp_json, imp_json, all_json))

707

print(" VUIDs checked in layer code: %04d explicit, %04d implicit, %04d total." % (exp_checks, imp_checks, all_checks))

708

print(" VUIDs tested in layer tests: %04d explicit, %04d implicit, %04d total." % (exp_tests, imp_tests, all_tests))

709

710

print("\nVUID check coverage")

711

print(" Explicit VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * exp_checks / exp_json), exp_checks, exp_json))

712

print(" Implicit VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * imp_checks / imp_json), imp_checks, imp_json))

713

print(" Overall VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * all_checks / all_json), all_checks, all_json))

Mike Weiblen

2017-02-03 12:44:53 -0700

[diff] [blame]

714

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

715

print("\nVUID test coverage")

716

print(" Explicit VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * exp_tests / exp_checks), exp_tests, exp_checks))

717

print(" Implicit VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * imp_tests / imp_checks), imp_tests, imp_checks))

718

print(" Overall VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * all_tests / all_checks), all_tests, all_checks))

Mike Weiblen

2017-02-03 12:44:53 -0700

[diff] [blame]

719

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

720

# Report status of a single VUID

721

if len(get_vuid_status) > 1:

722

print("\n\nChecking status of <%s>" % get_vuid_status);

723

if get_vuid_status not in val_json.all_vuids:

724

print(' Not a valid VUID string.')

725

else:

726

if get_vuid_status in val_source.explicit_vuids:

727

print(' Implemented!')

728

line_list = val_source.vuid_count_dict[get_vuid_status]['file_line']

729

for line in line_list:

730

print(' => %s' % line)

Dave Houlton

f93bbab

2018-06-26 17:21:12 -0600

[diff] [blame]

731

elif get_vuid_status in val_source.implicit_vuids:

732

print(' Implemented! (Implicit)')

733

line_list = val_source.vuid_count_dict[get_vuid_status]['file_line']

734

for line in line_list:

735

print(' => %s' % line)

Tobin Ehlis

9a68c98

2016-12-29 14:51:17 -0700

[diff] [blame]

736

else:

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

737

print(' Not implemented.')

Dave Houlton

f93bbab

2018-06-26 17:21:12 -0600

[diff] [blame]

738

if get_vuid_status in val_tests.all_vuids:

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

739

print(' Has a test!')

740

test_list = val_tests.vuid_to_tests[get_vuid_status]

741

for test in test_list:

742

print(' => %s' % test)

743

else:

744

print(' Not tested.')

Mike Weiblen

2017-02-03 12:44:53 -0700

[diff] [blame]

745

Dave Houlton

2018-05-29 13:00:42 -0600

[diff] [blame]

746

# Report unimplemented explicit VUIDs

747

if report_unimplemented:

748

unim_explicit = val_json.explicit_vuids - val_source.explicit_vuids

749

print("\n\n%d explicit VUID checks remain unimplemented:" % len(unim_explicit))

750

ulist = list(unim_explicit)

751

ulist.sort()

752

for vuid in ulist:

753

print(" => %s" % vuid)

# Consistency tests

if run_consistency:

print("\n\nRunning consistency tests...")

758

con = Consistency(val_json.all_vuids, val_source.all_vuids, val_tests.all_vuids)

759

ok = con.undef_vuids_in_layer_code()

760

ok &= con.undef_vuids_in_tests()

761

ok &= con.vuids_tested_not_checked()

762

763

if ok:

764

print(" OK! No inconsistencies found.")

765

766

# Output database in requested format(s)

767

db_out = OutputDatabase(val_json, val_source, val_tests)

if txt_out:

db_out.dump_txt()

if csv_out:

db_out.dump_csv()

if html_out:

db_out.dump_html()

Dave Houlton

2018-08-06 17:58:24 -0600

[diff] [blame^]

774

if header_out:

775

db_out.export_header()

Tobin Ehlis

20e3258

2016-12-05 14:50:03 -0700

[diff] [blame]

776

return result

Tobin Ehlis

2016-10-31 13:27:36 -0600

[diff] [blame]

777

778

if __name__ == "__main__":

Mark Lobodzinski

b44c54a

2017-06-12 12:02:38 -0600

[diff] [blame]

779

sys.exit(main(sys.argv[1:]))

Tobin Ehlis