From ebe10d511806cc0989f250509f9ab15ebbda0d5b Mon Sep 17 00:00:00 2001 From: Cedric Nugteren Date: Mon, 11 Sep 2017 20:40:37 +0200 Subject: [PATCH] Database now works with new format of clblast_[property] --- scripts/database/database.py | 29 +++++++++------------------ scripts/database/database/clblast.py | 23 +++++++++++---------- scripts/database/database/defaults.py | 10 ++++----- 3 files changed, 27 insertions(+), 35 deletions(-) diff --git a/scripts/database/database.py b/scripts/database/database.py index e398aa30..8f3ccce6 100755 --- a/scripts/database/database.py +++ b/scripts/database/database.py @@ -20,14 +20,6 @@ import database.defaults as defaults # Server storing a copy of the database DATABASE_SERVER_URL = "https://raw.githubusercontent.com/CNugteren/CLBlast-database/master/database.json" -# OpenCL vendor names and their short name -VENDOR_TRANSLATION_TABLE = { - "GenuineIntel": "Intel", - "Intel(R) Corporation": "Intel", - "Advanced Micro Devices, Inc.": "AMD", - "NVIDIA Corporation": "NVIDIA", -} - def remove_mismatched_arguments(database): """Checks for tuning results with mis-matched entries and removes them according to user preferences""" @@ -44,12 +36,14 @@ def remove_mismatched_arguments(database): for kernel_group_name, kernel_group in db.group_by(database["sections"], kernel_attributes): group_by_arguments = db.group_by(kernel_group, clblast.ARGUMENT_ATTRIBUTES) if len(group_by_arguments) != 1: - print("[database] WARNING: entries for a single kernel with multiple argument values " + str(kernel_group_name)) - print("[database] Either quit now, or remove all but one of the argument combinations below:") + print("[database] WARNING: entries for a single kernel with multiple argument values " + + str(kernel_group_name)) + print("[database] Either quit or remove all but one of the argument combinations below:") for index, (attribute_group_name, mismatching_entries) in enumerate(group_by_arguments): print("[database] %d: %s" % (index, attribute_group_name)) for attribute_group_name, mismatching_entries in group_by_arguments: - response = user_input("[database] Remove entries corresponding to %s, [y/n]? " % str(attribute_group_name)) + response = user_input("[database] Remove entries corresponding to %s, [y/n]? " % + str(attribute_group_name)) if response == "y": for entry in mismatching_entries: database["sections"].remove(entry) @@ -59,7 +53,8 @@ def remove_mismatched_arguments(database): for kernel_group_name, kernel_group in db.group_by(database["sections"], kernel_attributes): group_by_arguments = db.group_by(kernel_group, clblast.ARGUMENT_ATTRIBUTES) if len(group_by_arguments) != 1: - print("[database] ERROR: entries for a single kernel with multiple argument values " + str(kernel_group_name)) + print("[database] ERROR: entries for a single kernel with multiple argument values " + + str(kernel_group_name)) assert len(group_by_arguments) == 1 @@ -97,7 +92,8 @@ def main(argv): # Checks whether the command-line arguments are valid clblast_header = os.path.join(cl_args.clblast_root, "include", "clblast.h") # Not used but just for validation if not os.path.isfile(clblast_header): - raise RuntimeError("The path '" + cl_args.clblast_root + "' does not point to the root of the CLBlast library") + raise RuntimeError("The path '" + cl_args.clblast_root + + "' does not point to the root of the CLBlast library") if len(glob.glob(json_files)) < 1: print("[database] The path '" + cl_args.source_folder + "' does not contain any JSON files") @@ -115,11 +111,6 @@ def main(argv): sys.stdout.write("[database] Processing '" + file_json + "' ") # No newline printed imported_data = io.load_tuning_results(file_json) - # Fixes the problem that some vendors use multiple different names - for target in VENDOR_TRANSLATION_TABLE: - if imported_data["device_vendor"] == target: - imported_data["device_vendor"] = VENDOR_TRANSLATION_TABLE[target] - # Adds the new data to the database old_size = db.length(database) database = db.add_section(database, imported_data) @@ -136,7 +127,7 @@ def main(argv): # Removes database entries before continuing if cl_args.remove_device is not None: print("[database] Removing all results for device '%s'" % cl_args.remove_device) - remove_database_entries(database, {"device": cl_args.remove_device}) + remove_database_entries(database, {"clblast_device": cl_args.remove_device}) io.save_database(database, database_filename) # Retrieves the best performing results diff --git a/scripts/database/database/clblast.py b/scripts/database/database/clblast.py index 779dd76c..9ce502ee 100644 --- a/scripts/database/database/clblast.py +++ b/scripts/database/database/clblast.py @@ -13,8 +13,9 @@ DEVICE_TYPE_DEFAULT = "All" DEVICE_NAME_DEFAULT = "default" # List of attributes -DEVICE_TYPE_ATTRIBUTES = ["device_vendor", "device_type"] -DEVICE_ATTRIBUTES = ["device", "device_core_clock", "device_compute_units"] +DEVICE_TYPE_ATTRIBUTES = ["clblast_device_vendor", "clblast_device_type"] +DEVICE_ATTRIBUTES = ["clblast_device_name", "clblast_device_architecture", + "device_core_clock", "device_compute_units"] KERNEL_ATTRIBUTES = ["precision", "kernel_family"] ARGUMENT_ATTRIBUTES = ["arg_m", "arg_n", "arg_k", "arg_alpha", "arg_beta"] ATTRIBUTES = DEVICE_ATTRIBUTES + DEVICE_TYPE_ATTRIBUTES + KERNEL_ATTRIBUTES + ARGUMENT_ATTRIBUTES @@ -116,9 +117,9 @@ def print_cpp_database(database, output_dir): print("[database] No results found for %s:%s, retrieving defaults from %s:32" % (family_name, precision, family_name)) precision_database = [s for s in family_database if s["precision"] == "32" - and s["device_vendor"] == VENDOR_DEFAULT - and s["device_type"] == DEVICE_TYPE_DEFAULT - and s["device"] == DEVICE_NAME_DEFAULT] + and s["clblast_device_vendor"] == VENDOR_DEFAULT + and s["clblast_device_type"] == DEVICE_TYPE_DEFAULT + and s["clblast_device_name"] == DEVICE_NAME_DEFAULT] # Discovers the parameters for this kernel parameter_names = [] @@ -130,20 +131,20 @@ def print_cpp_database(database, output_dir): f.write(", {" + parameter_names_as_string + "}, {\n") # Loops over device vendors (e.g. AMD) - device_vendors = sorted(set([s["device_vendor"] for s in precision_database])) + device_vendors = sorted(set([s["clblast_device_vendor"] for s in precision_database])) for vendor in device_vendors: - vendor_database = [s for s in precision_database if s["device_vendor"] == vendor] + vendor_database = [s for s in precision_database if s["clblast_device_vendor"] == vendor] # Loops over device types (e.g. GPU) - device_types = sorted(set([s["device_type"] for s in vendor_database])) + device_types = sorted(set([s["clblast_device_type"] for s in vendor_database])) for device_type in device_types: - type_database = [s for s in vendor_database if s["device_type"] == device_type] + type_database = [s for s in vendor_database if s["clblast_device_type"] == device_type] f.write(get_cpp_device_vendor(vendor, device_type)) # Loops over every device of this vendor-type combination - devices = sorted(set([s["device"] for s in type_database])) + devices = sorted(set([s["clblast_device_name"] for s in type_database])) for device_name in devices: - device_database = [s for s in type_database if s["device"] == device_name] + device_database = [s for s in type_database if s["clblast_device_name"] == device_name] device_name_quoted = "\"%s\"," % device_name.strip() device_name_cpp = " { %-50s { " % device_name_quoted f.write(device_name_cpp) diff --git a/scripts/database/database/defaults.py b/scripts/database/database/defaults.py index 444c66df..3d11de34 100644 --- a/scripts/database/database/defaults.py +++ b/scripts/database/database/defaults.py @@ -14,9 +14,9 @@ import bests def set_default_device(section): """Sets the device name and parameters to some default values""" - section["device"] = clblast.DEVICE_NAME_DEFAULT - section["device_compute_units"] = 0 - section["device_core_clock"] = 0 + section["clblast_device_name"] = clblast.DEVICE_NAME_DEFAULT + section["clblast_device_compute_units"] = 0 + section["clblast_device_core_clock"] = 0 return section @@ -107,8 +107,8 @@ def calculate_defaults(database, verbose): if attribute != "results" and attribute != "group_identifier": default_section[attribute] = group[0][attribute] default_section = set_default_device(default_section) - default_section["device_vendor"] = clblast.VENDOR_DEFAULT - default_section["device_type"] = clblast.DEVICE_TYPE_DEFAULT + default_section["clblast_device_vendor"] = clblast.VENDOR_DEFAULT + default_section["clblast_device_type"] = clblast.DEVICE_TYPE_DEFAULT default_section["results"] = [{"time": 0.0, "parameters": default_parameters}] default_sections["sections"].append(default_section)