mirror of
https://github.com/CNugteren/CLBlast.git
synced 2024-07-04 21:36:57 +02:00
Refactored the Python database script: separated functionality in modules, now complies to the PEP8 style, added proper command-line argument parsing, and cleaned-up
This commit is contained in:
parent
b33bec4a59
commit
622682ffe3
356
scripts/database/database.py
Normal file → Executable file
356
scripts/database/database.py
Normal file → Executable file
|
@ -1,326 +1,112 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# ==================================================================================================
|
||||
# This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
|
||||
# project loosely follows the Google C++ styleguide and uses a max-width of 100 characters per line.
|
||||
# This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This file follows the
|
||||
# PEP8 Python style guide and uses a max-width of 120 characters per line.
|
||||
#
|
||||
# Author(s):
|
||||
# Cedric Nugteren <www.cedricnugteren.nl>
|
||||
#
|
||||
# ==================================================================================================
|
||||
|
||||
# System modules
|
||||
import sys
|
||||
import os.path
|
||||
import glob
|
||||
import re
|
||||
import json
|
||||
try:
|
||||
from urllib.request import urlopen # Python 3
|
||||
except ImportError:
|
||||
from urllib2 import urlopen # Python 2
|
||||
import argparse
|
||||
|
||||
# Additional modules
|
||||
import pandas as pd
|
||||
print("## Using pandas version "+pd.__version__+", requires at least 0.17.0")
|
||||
|
||||
import database.io as io
|
||||
import database.db as db
|
||||
import database.clblast as clblast
|
||||
import database.bests as bests
|
||||
import database.defaults as defaults
|
||||
|
||||
# Server storing a copy of the database
|
||||
DATABASE_SERVER_URL = "http://www.cedricnugteren.nl/tuning/clblast.db"
|
||||
|
||||
# Constants
|
||||
VENDOR_DEFAULT = "default"
|
||||
DEVICETYPE_DEFAULT = "All"
|
||||
DEVICENAME_DEFAULT = "default"
|
||||
|
||||
# Attributes
|
||||
DEVICETYPE_ATTRIBUTES = ["device_vendor", "device_type"]
|
||||
DEVICE_ATTRIBUTES = ["device", "device_core_clock", "device_compute_units"]
|
||||
KERNEL_ATTRIBUTES = ["precision", "kernel_family"]
|
||||
ARGUMENT_ATTRIBUTES = ["arg_m", "arg_n", "arg_k", "arg_alpha", "arg_beta"]
|
||||
ATTRIBUTES = DEVICE_ATTRIBUTES + DEVICETYPE_ATTRIBUTES + KERNEL_ATTRIBUTES + ARGUMENT_ATTRIBUTES
|
||||
|
||||
# OpenCL vendor names and their short name
|
||||
VENDOR_NAMES = { "device_vendor": {
|
||||
VENDOR_TRANSLATION_TABLE = {"device_vendor": {
|
||||
"GenuineIntel": "Intel",
|
||||
"Intel(R) Corporation": "Intel",
|
||||
"Advanced Micro Devices, Inc.": "AMD",
|
||||
"NVIDIA Corporation": "NVIDIA",
|
||||
}}
|
||||
|
||||
# Pandas options
|
||||
pd.set_option('display.width', 1000)
|
||||
|
||||
# ==================================================================================================
|
||||
# Database operations
|
||||
# ==================================================================================================
|
||||
def main(argv):
|
||||
|
||||
# Downloads the database and save it to disk
|
||||
def DownloadDatabase(filename):
|
||||
print("## Downloading database from '"+DATABASE_SERVER_URL+"'...")
|
||||
df = urlopen(DATABASE_SERVER_URL)
|
||||
output = open(file_db,'wb')
|
||||
output.write(df.read())
|
||||
output.close()
|
||||
# Parses the command-line arguments
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("source_folder", help="The folder with JSON files to parse to add to the database")
|
||||
parser.add_argument("clblast_root", help="Root of the CLBlast sources")
|
||||
parser.add_argument("-v", "--verbose", action="store_true", help="Increase verbosity of the script")
|
||||
cl_args = parser.parse_args(argv)
|
||||
|
||||
# Loads the database from disk
|
||||
def LoadDatabase(filename):
|
||||
return pd.read_pickle(filename)
|
||||
# Parses the path arguments
|
||||
database_filename = os.path.join(cl_args.clblast_root, "scripts", "database", "database.db")
|
||||
json_files = os.path.join(cl_args.source_folder, "*.json")
|
||||
cpp_database_path = os.path.join(cl_args.clblast_root, "src", "database", "kernels")
|
||||
|
||||
# Saves the database to disk
|
||||
def SaveDatabase(df, filename):
|
||||
df.to_pickle(filename)
|
||||
# Checks whether the command-line arguments are valid
|
||||
clblast_header = os.path.join(cl_args.clblast_root, "include", "clblast.h") # Not used but just for validation
|
||||
if not os.path.isfile(clblast_header):
|
||||
raise RuntimeError("The path '" + cl_args.clblast_root + "' does not point to the root of the CLBlast library")
|
||||
if len(glob.glob(json_files)) < 1:
|
||||
print("[database] The path '" + cl_args.source_folder + "' does not contain any JSON files")
|
||||
|
||||
# Loads JSON data from file
|
||||
def ImportDataFromFile(filename):
|
||||
with open(filename) as f:
|
||||
data = json.load(f)
|
||||
json_data = pd.DataFrame(data)
|
||||
df = pd.io.json.json_normalize(json_data["results"])
|
||||
for attribute in ATTRIBUTES:
|
||||
if attribute == "kernel_family":
|
||||
df[attribute] = re.sub(r'_\d+', '', data[attribute])
|
||||
elif attribute in data:
|
||||
df[attribute] = data[attribute]
|
||||
else:
|
||||
df[attribute] = 0
|
||||
return df
|
||||
# Pandas options
|
||||
pd.set_option('display.width', 1000)
|
||||
if cl_args.verbose:
|
||||
print("[database] Using pandas version " + pd.__version__)
|
||||
|
||||
# Returns the row-wise concatenation of two dataframes
|
||||
def ConcatenateData(df1, df2):
|
||||
return pd.concat([df1, df2])
|
||||
# Downloads the database if a local copy is not present
|
||||
if not os.path.isfile(database_filename):
|
||||
io.download_database(database_filename, DATABASE_SERVER_URL)
|
||||
|
||||
# Removes duplicates from a dataframe
|
||||
def RemoveDuplicates(df):
|
||||
return df.drop_duplicates()
|
||||
# Loads the database from disk
|
||||
database = io.load_database(database_filename)
|
||||
|
||||
# database = database[(database["device"] != "AMD Radeon R9 M370X Compute Engine") | (database["kernel_family"] != "xgemm") | (database["precision"] != "32")]
|
||||
def RemoveEntriesByDevice(df, devicename):
|
||||
return df[df["device"] != devicename]
|
||||
# Loops over all JSON files in the supplied folder
|
||||
for file_json in glob.glob(json_files):
|
||||
|
||||
def RemoveEntriesByKernelFamily(df, familyname):
|
||||
return df[df["kernel_family"] != familyname]
|
||||
# Loads the newly imported data
|
||||
sys.stdout.write("[database] Processing '"+file_json+"' ") # No newline printed
|
||||
imported_data = io.load_json_to_pandas(file_json)
|
||||
|
||||
def GetEntriesByField(df, field, value):
|
||||
return df[df[field] == value]
|
||||
# Fixes the problem that some vendors use multiple different names
|
||||
imported_data = db.find_and_replace(imported_data, VENDOR_TRANSLATION_TABLE)
|
||||
|
||||
# Example usage:
|
||||
# df = UpdateDatabase(df, (df["kernel_family"] == "xdot") & (df["arg_n"] == "67108864"), "arg_n", "2097152")
|
||||
def UpdateDatabase(df, condition, field, value):
|
||||
df.loc[condition, field] = value
|
||||
return df
|
||||
# Adds the new data to the database
|
||||
old_size = len(database.index)
|
||||
database = db.concatenate_database(database, imported_data)
|
||||
database = db.remove_duplicates(database)
|
||||
new_size = len(database.index)
|
||||
print("with " + str(new_size - old_size) + " new items") # Newline printed here
|
||||
|
||||
# Fixes the problem that some vendors use multiple different names
|
||||
def SanitizeVendorNames(df):
|
||||
df = df.replace(VENDOR_NAMES)
|
||||
return df
|
||||
# Stores the modified database back to disk
|
||||
if len(glob.glob(json_files)) >= 1:
|
||||
io.save_database(database, database_filename)
|
||||
|
||||
# Retrieves the results with the lowest execution times
|
||||
def GetBestResults(df):
|
||||
dfbest = pd.DataFrame()
|
||||
grouped = df.groupby(ATTRIBUTES+["kernel"])
|
||||
for name, dfgroup in grouped:
|
||||
besttime = dfgroup["time"].min()
|
||||
bestcase = dfgroup[dfgroup["time"] == besttime].iloc[0]
|
||||
dfbest = dfbest.append(bestcase, ignore_index=True)
|
||||
return dfbest
|
||||
# Optional: update the database here. Default is disabled, code below is just an example
|
||||
if False: # TODO: Use command-line arguments to enable updates in a flexible way
|
||||
database = db.update_database(database,
|
||||
((database["kernel"] == "CopyMatrixFast") &
|
||||
(database["precision"] == "3232")),
|
||||
"arg_alpha", "2+0.5i")
|
||||
io.save_database(database, database_filename)
|
||||
|
||||
# Sets defaults for devices of the same type/vendor based on the smallest values of all know
|
||||
# entries. The average might be better for performance but some parameters might not be supported
|
||||
# on other devices.
|
||||
def CalculateDefaults(df):
|
||||
dfdefault = pd.DataFrame()
|
||||
# Retrieves the best performing results
|
||||
print("[database] Calculating the best results per device/kernel...")
|
||||
database_best_results = bests.get_best_results(database)
|
||||
|
||||
# Defaults per type/vendor
|
||||
groups = df.groupby(DEVICETYPE_ATTRIBUTES+KERNEL_ATTRIBUTES+ARGUMENT_ATTRIBUTES+["kernel"])
|
||||
for name, dfgroup in groups:
|
||||
default_values = dfgroup.min(axis=0)
|
||||
default_values["device"] = DEVICENAME_DEFAULT
|
||||
default_values["device_compute_units"] = 0
|
||||
default_values["device_core_clock"] = 0
|
||||
default_values["time"] = 0.0
|
||||
dfdefault = dfdefault.append(default_values, ignore_index=True)
|
||||
|
||||
# Checks for mis-matched arguments
|
||||
groups = dfdefault.groupby(DEVICETYPE_ATTRIBUTES+KERNEL_ATTRIBUTES+["kernel"])
|
||||
for name, dfgroup in groups:
|
||||
if len(dfgroup) != 1:
|
||||
description = dfgroup["kernel"].min() + " " + dfgroup["device_vendor"].min()
|
||||
print("[WARNING] Entries for a single kernel with multiple argument values: " + description)
|
||||
|
||||
# Defaults in general
|
||||
groups = df.groupby(KERNEL_ATTRIBUTES+ARGUMENT_ATTRIBUTES+["kernel"])
|
||||
for name, dfgroup in groups:
|
||||
default_values = dfgroup.min(axis=0)
|
||||
default_values["device_vendor"] = VENDOR_DEFAULT
|
||||
default_values["device_type"] = DEVICETYPE_DEFAULT
|
||||
default_values["device"] = DEVICENAME_DEFAULT
|
||||
default_values["device_compute_units"] = 0
|
||||
default_values["device_core_clock"] = 0
|
||||
default_values["time"] = 0.0
|
||||
dfdefault = dfdefault.append(default_values, ignore_index=True)
|
||||
|
||||
# Database with both types of defaults only
|
||||
return dfdefault
|
||||
# Determines the defaults for other vendors and per vendor
|
||||
database_defaults = defaults.calculate_defaults(database_best_results)
|
||||
database_best_results = db.concatenate_database(database_best_results, database_defaults)
|
||||
|
||||
# ==================================================================================================
|
||||
# C++ header generation
|
||||
# ==================================================================================================
|
||||
# Outputs the database as a C++ database
|
||||
print("[database] Producing a C++ database in '" + cpp_database_path + "'...")
|
||||
clblast.print_cpp_database(database_best_results, cpp_database_path)
|
||||
|
||||
# The C++ header
|
||||
def GetHeader(family):
|
||||
return("""
|
||||
// =================================================================================================
|
||||
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
|
||||
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
|
||||
// width of 100 characters per line.
|
||||
//
|
||||
// Author(s):
|
||||
// Database generator <database.py>
|
||||
//
|
||||
// This file populates the database with best-found tuning parameters for the '%s' kernels.
|
||||
//
|
||||
// =================================================================================================
|
||||
print("[database] All done")
|
||||
|
||||
namespace clblast {
|
||||
// ================================================================================================="""
|
||||
% family.title())
|
||||
|
||||
# The C++ footer
|
||||
def GetFooter():
|
||||
return("\n} // namespace clblast\n")
|
||||
|
||||
# The start of a new C++ precision entry
|
||||
def GetPrecision(family, precision):
|
||||
precisionstring = ""
|
||||
if precision == "16":
|
||||
precisionstring = "Half"
|
||||
elif precision == "32":
|
||||
precisionstring = "Single"
|
||||
elif precision == "64":
|
||||
precisionstring = "Double"
|
||||
elif precision == "3232":
|
||||
precisionstring = "ComplexSingle"
|
||||
elif precision == "6464":
|
||||
precisionstring = "ComplexDouble"
|
||||
else:
|
||||
print("[ERROR] Unknown precision")
|
||||
sys.exit()
|
||||
return("\n\nconst Database::DatabaseEntry Database::%s%s = {\n \"%s\", Precision::k%s, {\n"
|
||||
% (family.title(), precisionstring, family.title(), precisionstring))
|
||||
|
||||
# The C++ device type and vendor
|
||||
def GetDeviceVendor(vendor, devtype):
|
||||
if vendor == VENDOR_DEFAULT and devtype == DEVICETYPE_DEFAULT:
|
||||
return(" { // Default\n kDeviceType%s, \"%s\", {\n" % (devtype, vendor))
|
||||
return(" { // %s %ss\n kDeviceType%s, \"%s\", {\n" % (vendor, devtype, devtype[0].upper() + devtype[1:], vendor))
|
||||
|
||||
# Prints the data to a C++ database
|
||||
def PrintData(df, outputdir):
|
||||
|
||||
# Iterates over the kernel families: creates a new file per family
|
||||
for family, dffamily in df.groupby(["kernel_family"]):
|
||||
dffamily = dffamily.dropna(axis=1, how='all')
|
||||
f = open(os.path.join(outputdir, family+'.hpp'), 'w+')
|
||||
f.write(GetHeader(family))
|
||||
|
||||
# Loops over the different entries for this family and prints their headers
|
||||
for precision, dfprecision in dffamily.groupby(["precision"]):
|
||||
f.write(GetPrecision(family, precision))
|
||||
for vendor, dfvendor in dfprecision.groupby(["device_vendor"]):
|
||||
for devtype, dfdevtype in dfvendor.groupby(["device_type"]):
|
||||
f.write(GetDeviceVendor(vendor, devtype))
|
||||
for device, dfdevice in dfdevtype.groupby(["device"]):
|
||||
devicename = "\"%s\"," % device
|
||||
f.write(" { %-50s { " % devicename)
|
||||
|
||||
# Collects the paramaters for this case and prints them
|
||||
parameters = []
|
||||
for kernel, dfkernel in dfdevice.groupby(["kernel"]):
|
||||
dfkernel = dfkernel.dropna(axis=1)
|
||||
col_names = [col for col in list(dfkernel) if col.startswith('parameters.') and col != "parameters.PRECISION"]
|
||||
parameters += ["{\"%s\",%d}" % (p.replace("parameters.",""), dfkernel[p].iloc[0]) for p in col_names]
|
||||
f.write(", ".join(parameters))
|
||||
f.write(" } },\n")
|
||||
|
||||
# Prints the footers
|
||||
f.write(" }\n },\n")
|
||||
f.write(" }\n};\n\n// =================================================================================================")
|
||||
f.write(GetFooter())
|
||||
|
||||
# ==================================================================================================
|
||||
# Command-line arguments parsing and verification
|
||||
# ==================================================================================================
|
||||
|
||||
# Checks for the number of command-line arguments
|
||||
if len(sys.argv) != 3:
|
||||
print("[ERROR] Usage: database.py <folder_with_json_files> <root_of_clblast>")
|
||||
sys.exit()
|
||||
|
||||
# Parses the command-line arguments
|
||||
path_json = sys.argv[1]
|
||||
path_clblast = sys.argv[2]
|
||||
file_db = os.path.join(path_clblast, "scripts", "database", "database.db")
|
||||
glob_json = os.path.join(path_json, "*.json")
|
||||
|
||||
# Checks whether the command-line arguments are valid; exists otherwise
|
||||
clblast_h = os.path.join(path_clblast, "include", "clblast.h") # Not used but just for validation
|
||||
if not os.path.isfile(clblast_h):
|
||||
print("[ERROR] The path '"+path_clblast+"' does not point to the root of the CLBlast library")
|
||||
sys.exit()
|
||||
if len(glob.glob(glob_json)) < 1:
|
||||
print("## The path '"+path_json+"' does not contain any JSON files")
|
||||
|
||||
# ==================================================================================================
|
||||
# The main body of the script
|
||||
# ==================================================================================================
|
||||
|
||||
# Downloads the database if a local copy is not present
|
||||
db_exists = os.path.isfile(file_db)
|
||||
if not db_exists:
|
||||
DownloadDatabase(file_db)
|
||||
|
||||
# Loads the database from disk
|
||||
print("## Loading the database from disk...")
|
||||
database = LoadDatabase(file_db)
|
||||
|
||||
# Loops over all JSON files in the supplied folder
|
||||
for file_json in glob.glob(glob_json):
|
||||
|
||||
# Loads the newly imported data
|
||||
sys.stdout.write("## Processing '"+file_json+"' ")
|
||||
imported_data = ImportDataFromFile(file_json)
|
||||
imported_data = SanitizeVendorNames(imported_data)
|
||||
|
||||
# Adds the new data to the database
|
||||
old_size = len(database.index)
|
||||
database = ConcatenateData(database, imported_data)
|
||||
database = RemoveDuplicates(database)
|
||||
new_size = len(database.index)
|
||||
print("with "+str(new_size-old_size)+" new items")
|
||||
|
||||
# Stores the modified database back to disk
|
||||
if len(glob.glob(glob_json)) >= 1:
|
||||
print("## Storing the database to disk...")
|
||||
SaveDatabase(database, file_db)
|
||||
|
||||
# Optional: update the database here. Default is disabled, code below is just an example
|
||||
if False:
|
||||
database = UpdateDatabase(database, ((database["kernel"] == "CopyMatrixFast") & (database["precision"] == "3232")), "arg_alpha", "2+0.5i")
|
||||
SaveDatabase(database, file_db)
|
||||
|
||||
# Retrieves the best performing results
|
||||
print("## Calculating the best results per device/kernel...")
|
||||
bests = GetBestResults(database)
|
||||
|
||||
# Determines the defaults for other vendors and per vendor
|
||||
defaults = CalculateDefaults(bests)
|
||||
bests = ConcatenateData(bests, defaults)
|
||||
|
||||
# Outputs the data as a C++ database
|
||||
path_cpp_database = os.path.join(path_clblast, "src", "database", "kernels")
|
||||
print("## Producing a C++ database in '"+path_cpp_database+"'...")
|
||||
PrintData(bests, path_cpp_database)
|
||||
|
||||
print("## All done")
|
||||
|
||||
# ==================================================================================================
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[1:])
|
||||
|
|
0
scripts/database/database/__init__.py
Normal file
0
scripts/database/database/__init__.py
Normal file
20
scripts/database/database/bests.py
Normal file
20
scripts/database/database/bests.py
Normal file
|
@ -0,0 +1,20 @@
|
|||
|
||||
# This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This file follows the
|
||||
# PEP8 Python style guide and uses a max-width of 120 characters per line.
|
||||
#
|
||||
# Author(s):
|
||||
# Cedric Nugteren <www.cedricnugteren.nl>
|
||||
|
||||
import pandas as pd
|
||||
import clblast
|
||||
|
||||
|
||||
def get_best_results(df):
|
||||
"""Retrieves the results with the lowests execution times"""
|
||||
database_bests = pd.DataFrame()
|
||||
database_entries = df.groupby(clblast.ATTRIBUTES + ["kernel"])
|
||||
for name, database_entry in database_entries:
|
||||
best_time = database_entry["time"].min()
|
||||
best_parameters = database_entry[database_entry["time"] == best_time].iloc[0]
|
||||
database_bests = database_bests.append(best_parameters, ignore_index=True)
|
||||
return database_bests
|
132
scripts/database/database/clblast.py
Normal file
132
scripts/database/database/clblast.py
Normal file
|
@ -0,0 +1,132 @@
|
|||
|
||||
# This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This file follows the
|
||||
# PEP8 Python style guide and uses a max-width of 120 characters per line.
|
||||
#
|
||||
# Author(s):
|
||||
# Cedric Nugteren <www.cedricnugteren.nl>
|
||||
|
||||
import os
|
||||
|
||||
# Constants from the C++ code
|
||||
VENDOR_DEFAULT = "default"
|
||||
DEVICE_TYPE_DEFAULT = "All"
|
||||
DEVICE_NAME_DEFAULT = "default"
|
||||
|
||||
# List of attributes
|
||||
DEVICE_TYPE_ATTRIBUTES = ["device_vendor", "device_type"]
|
||||
DEVICE_ATTRIBUTES = ["device", "device_core_clock", "device_compute_units"]
|
||||
KERNEL_ATTRIBUTES = ["precision", "kernel_family"]
|
||||
ARGUMENT_ATTRIBUTES = ["arg_m", "arg_n", "arg_k", "arg_alpha", "arg_beta"]
|
||||
ATTRIBUTES = DEVICE_ATTRIBUTES + DEVICE_TYPE_ATTRIBUTES + KERNEL_ATTRIBUTES + ARGUMENT_ATTRIBUTES
|
||||
|
||||
|
||||
def precision_to_string(precision):
|
||||
"""Translates a precision number (represented as Python string) into a descriptive string"""
|
||||
if precision == "16":
|
||||
return "Half"
|
||||
elif precision == "32":
|
||||
return "Single"
|
||||
elif precision == "64":
|
||||
return "Double"
|
||||
elif precision == "3232":
|
||||
return "ComplexSingle"
|
||||
elif precision == "6464":
|
||||
return "ComplexDouble"
|
||||
else:
|
||||
raise("Unknown precision: " + precision)
|
||||
|
||||
|
||||
def get_cpp_separator():
|
||||
"""Retrieves a C++ comment separator"""
|
||||
return "// ================================================================================================="
|
||||
|
||||
|
||||
def get_cpp_header(family):
|
||||
"""Retrieves the C++ header"""
|
||||
return ("\n" + get_cpp_separator() + """
|
||||
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
|
||||
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
|
||||
// width of 100 characters per line.
|
||||
//
|
||||
// Author(s):
|
||||
// Database generator <database.py>
|
||||
//
|
||||
// This file populates the database with best-found tuning parameters for the '%s' kernels.
|
||||
//\n"""
|
||||
% family.title() + get_cpp_separator() + "\n\nnamespace clblast {\n" + get_cpp_separator())
|
||||
|
||||
|
||||
def get_cpp_footer():
|
||||
"""Retrieves the C++ footer"""
|
||||
return "\n} // namespace clblast\n"
|
||||
|
||||
|
||||
def get_cpp_precision(family, precision):
|
||||
"""Retrieves the C++ code for the start of a new precision"""
|
||||
precision_string = precision_to_string(precision)
|
||||
return("\n\nconst Database::DatabaseEntry Database::%s%s = {\n \"%s\", Precision::k%s, {\n"
|
||||
% (family.title(), precision_string, family.title(), precision_string))
|
||||
|
||||
|
||||
def get_cpp_device_vendor(vendor, device_type):
|
||||
"""Retrieves the C++ code for the (default) vendor and device type"""
|
||||
if vendor == VENDOR_DEFAULT and device_type == DEVICE_TYPE_DEFAULT:
|
||||
return " { // Default\n kDeviceType%s, \"%s\", {\n" % (device_type, vendor)
|
||||
device_type_caps = device_type[0].upper() + device_type[1:]
|
||||
return " { // %s %ss\n kDeviceType%s, \"%s\", {\n" % (vendor, device_type, device_type_caps, vendor)
|
||||
|
||||
|
||||
def print_cpp_database(database, output_dir):
|
||||
"""Outputs the database as C++ code"""
|
||||
|
||||
# Iterates over the kernel families
|
||||
for family_name, family_database in database.groupby(["kernel_family"]):
|
||||
family_database = family_database.dropna(axis=1, how='all')
|
||||
|
||||
# Opens a new file for each kernel family
|
||||
full_path = os.path.join(output_dir, family_name+'.hpp')
|
||||
with open(full_path, 'w+') as f:
|
||||
f.write(get_cpp_header(family_name))
|
||||
|
||||
# Loops over the different precision (e.g. 16, 32, 3232, 64, 6464)
|
||||
for precision, precision_database in family_database.groupby(["precision"]):
|
||||
f.write(get_cpp_precision(family_name, precision))
|
||||
|
||||
# Loops over a combination of device vendors and device types (e.g. AMD GPU)
|
||||
for vendor, vendor_database in precision_database.groupby(["device_vendor"]):
|
||||
for device_type, device_type_database in vendor_database.groupby(["device_type"]):
|
||||
f.write(get_cpp_device_vendor(vendor, device_type))
|
||||
|
||||
# Loops over every device of this vendor-type combination
|
||||
for device_name, device_database in device_type_database.groupby(["device"]):
|
||||
device_name_quoted = "\"%s\"," % device_name
|
||||
device_name_cpp = " { %-50s { " % device_name_quoted
|
||||
f.write(device_name_cpp)
|
||||
|
||||
# Collects the parameters for this entry
|
||||
parameters = []
|
||||
for kernel, kernel_database in device_database.groupby(["kernel"]):
|
||||
kernel_database = kernel_database.dropna(axis=1)
|
||||
|
||||
# Only consider the actual parameters, not the precision
|
||||
def is_parameter(column):
|
||||
return column.startswith('parameters.') and column != "parameters.PRECISION"
|
||||
column_names = [col for col in list(kernel_database) if is_parameter(col)]
|
||||
|
||||
for p in column_names:
|
||||
parameter_name = p.replace("parameters.", "")
|
||||
parameter_value = int(kernel_database[p].iloc[0])
|
||||
parameters.append("{\"" + parameter_name + "\"," + str(parameter_value) + "}")
|
||||
|
||||
# Prints the entry
|
||||
f.write(", ".join(parameters))
|
||||
f.write(" } },\n")
|
||||
|
||||
# Prints the vendor-type combination footer
|
||||
f.write(" }\n },\n")
|
||||
|
||||
# Prints the precision footer
|
||||
f.write(" }\n};\n\n" + get_cpp_separator())
|
||||
|
||||
# Prints the file footer
|
||||
f.write(get_cpp_footer())
|
50
scripts/database/database/db.py
Normal file
50
scripts/database/database/db.py
Normal file
|
@ -0,0 +1,50 @@
|
|||
|
||||
# This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This file follows the
|
||||
# PEP8 Python style guide and uses a max-width of 120 characters per line.
|
||||
#
|
||||
# Author(s):
|
||||
# Cedric Nugteren <www.cedricnugteren.nl>
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def get_entries_by_field(database, field, value):
|
||||
"""Retrieves entries from the database with a specific value for a given field"""
|
||||
return database[database[field] == value]
|
||||
|
||||
|
||||
def concatenate_database(database1, database2):
|
||||
"""Concatenates two databases row-wise and returns the result"""
|
||||
return pd.concat([database1, database2])
|
||||
|
||||
|
||||
def remove_duplicates(database):
|
||||
"""Removes duplicates from a database"""
|
||||
return database.drop_duplicates()
|
||||
|
||||
|
||||
def find_and_replace(database, dictionary):
|
||||
"""Finds and replaces entries in a database based on a dictionary. Example:
|
||||
dictionary = { "key_to_edit": { find1: replace1, find2, replace2 } }"""
|
||||
return database.replace(dictionary)
|
||||
|
||||
|
||||
def remove_entries_by_key_value(database, key, value):
|
||||
"""Removes entries in the databased which have a specific value for a given key"""
|
||||
return database[database[key] != value]
|
||||
|
||||
|
||||
def remove_entries_by_device(database, device_name):
|
||||
"""Shorthand for the above, specifically removes entries for a given device"""
|
||||
return remove_entries_by_key_value(database, "device", device_name)
|
||||
|
||||
|
||||
def remove_entries_by_kernel_family(database, kernel_family_name):
|
||||
"""Shorthand for the above, specifically removes entries for a given kernel family"""
|
||||
return remove_entries_by_key_value(database, "kernel_family", kernel_family_name)
|
||||
|
||||
|
||||
def update_database(database, condition, field, value):
|
||||
"""Updates the database by writing a specific value to a given field, given certain conditions"""
|
||||
database.loc[condition, field] = value
|
||||
return database
|
58
scripts/database/database/defaults.py
Normal file
58
scripts/database/database/defaults.py
Normal file
|
@ -0,0 +1,58 @@
|
|||
|
||||
# This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This file follows the
|
||||
# PEP8 Python style guide and uses a max-width of 120 characters per line.
|
||||
#
|
||||
# Author(s):
|
||||
# Cedric Nugteren <www.cedricnugteren.nl>
|
||||
|
||||
import pandas as pd
|
||||
import clblast
|
||||
|
||||
|
||||
def set_default_device(database_entry):
|
||||
"""Sets the device name and parameters to some default values"""
|
||||
database_entry["device"] = clblast.DEVICE_NAME_DEFAULT
|
||||
database_entry["device_compute_units"] = 0
|
||||
database_entry["device_core_clock"] = 0
|
||||
return database_entry
|
||||
|
||||
|
||||
def set_default_time(database_entry):
|
||||
"""Sets the execution time to some default value"""
|
||||
database_entry["time"] = 0.0
|
||||
return database_entry
|
||||
|
||||
|
||||
def calculate_defaults(df):
|
||||
"""# Sets defaults for devices of the same type/vendor based on the smallest values of all known entries. The average
|
||||
might be better for performance but some parameters might not be supported on other devices."""
|
||||
database_defaults = pd.DataFrame()
|
||||
|
||||
# Defaults per combination of device vendors and device types (e.g. AMD GPU)
|
||||
database_type_vendor = df.groupby(clblast.DEVICE_TYPE_ATTRIBUTES + clblast.KERNEL_ATTRIBUTES + ["kernel"] +
|
||||
clblast.ARGUMENT_ATTRIBUTES)
|
||||
for group_name, database_group in database_type_vendor:
|
||||
default_values = database_group.min(axis=0)
|
||||
default_values = set_default_device(default_values)
|
||||
default_values = set_default_time(default_values)
|
||||
database_defaults = database_defaults.append(default_values, ignore_index=True)
|
||||
|
||||
# Checks for mis-matched arguments
|
||||
groups = database_defaults.groupby(clblast.DEVICE_TYPE_ATTRIBUTES + clblast.KERNEL_ATTRIBUTES + ["kernel"])
|
||||
for group_name, database_group in groups:
|
||||
if len(database_group) != 1:
|
||||
description = database_group["kernel"].min() + " " + database_group["device_vendor"].min()
|
||||
print("[WARNING] Entries for a single kernel with multiple argument values: " + description)
|
||||
|
||||
# Defaults over all device types and vendors
|
||||
groups = df.groupby(clblast.KERNEL_ATTRIBUTES + ["kernel"] + clblast.ARGUMENT_ATTRIBUTES)
|
||||
for group_name, database_group in groups:
|
||||
default_values = database_group.min(axis=0)
|
||||
default_values["device_vendor"] = clblast.VENDOR_DEFAULT
|
||||
default_values["device_type"] = clblast.DEVICE_TYPE_DEFAULT
|
||||
default_values = set_default_device(default_values)
|
||||
default_values = set_default_time(default_values)
|
||||
database_defaults = database_defaults.append(default_values, ignore_index=True)
|
||||
|
||||
# Database with both types of defaults only
|
||||
return database_defaults
|
58
scripts/database/database/io.py
Normal file
58
scripts/database/database/io.py
Normal file
|
@ -0,0 +1,58 @@
|
|||
|
||||
# This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This file follows the
|
||||
# PEP8 Python style guide and uses a max-width of 120 characters per line.
|
||||
#
|
||||
# Author(s):
|
||||
# Cedric Nugteren <www.cedricnugteren.nl>
|
||||
|
||||
import re
|
||||
import json
|
||||
|
||||
try:
|
||||
from urllib.request import urlopen # Python 3
|
||||
except ImportError:
|
||||
from urllib2 import urlopen # Python 2
|
||||
|
||||
import pandas as pd
|
||||
|
||||
import clblast
|
||||
|
||||
|
||||
def download_database(filename, database_url):
|
||||
"""Downloads a database and saves it to disk"""
|
||||
print("[database] Downloading database from '" + database_url + "'...")
|
||||
database = urlopen(database_url)
|
||||
with open(filename, 'wb') as f:
|
||||
f.write(database.read())
|
||||
|
||||
|
||||
def load_database(filename):
|
||||
"""Loads a database from disk"""
|
||||
print("[database] Loading database from '" + filename + "'")
|
||||
return pd.read_pickle(filename)
|
||||
|
||||
|
||||
def save_database(database, filename):
|
||||
"""Saves a database to disk"""
|
||||
print("[database] Saving database to '" + filename + "'")
|
||||
database.to_pickle(filename)
|
||||
|
||||
|
||||
def load_json_to_pandas(filename):
|
||||
"""Loads JSON data from file and converts it to a pandas database"""
|
||||
with open(filename) as f:
|
||||
json_data = json.load(f)
|
||||
|
||||
# Gathers all results and stores them in a new database
|
||||
json_database = pd.DataFrame(json_data)
|
||||
new_database = pd.io.json.json_normalize(json_database["results"])
|
||||
|
||||
# Sets the common attributes to each entry in the results
|
||||
for attribute in clblast.ATTRIBUTES:
|
||||
if attribute == "kernel_family":
|
||||
new_database[attribute] = re.sub(r'_\d+', '', json_data[attribute])
|
||||
elif attribute in json_data:
|
||||
new_database[attribute] = json_data[attribute]
|
||||
else:
|
||||
new_database[attribute] = 0 # For example a parameters that was not used by this kernel
|
||||
return new_database
|
Loading…
Reference in a new issue