Skip to content

ci: Update result tracking with new golden format #2706

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rjodinchr wants to merge 1 commit into
base: main
Choose a base branch
from
Open

ci: Update result tracking with new golden format #2706

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion ci/compare_results.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ def process_all_results(results_dir, golden_data):

cmd = os.path.basename(data.get("cmd"))
results = data.get("results", {})
args = data.get("args", "")

if not cmd:
print(f"File {result_file} is missing the 'cmd' key. Skipping.")
Expand All @@ -84,8 +85,13 @@ def process_all_results(results_dir, golden_data):
has_error = True
continue

if args not in golden_data[cmd]:
missing_refs.append(f"args '{args}' not found in golden reference for cmd '{cmd}'.")
has_error = True
continue

# Check the specific results against the reference
subset_error = compare_test_subset(cmd, results, golden_data[cmd], differences, missing_refs)
subset_error = compare_test_subset(cmd, results, golden_data[cmd][args], differences, missing_refs)
if subset_error:
has_error = True

Expand Down
132 changes: 127 additions & 5 deletions ci/pocl/golden.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,131 @@
{
"test_bruteforce": {
"--wimpy -1": {
"math_edge_cases": "fail",
"acos": "pass",
"acosh": "fail",
"acospi": "fail",
"asin": "pass",
"asinh": "fail",
"asinpi": "fail",
"atan": "pass",
"atanh": "fail",
"atanpi": "fail",
"atan2": "fail",
"atan2pi": "fail",
"cbrt": "fail",
"ceil": "pass",
"copysign": "pass",
"cos": "pass",
"cosh": "pass",
"cospi": "fail",
"exp": "pass",
"exp2": "pass",
"exp10": "pass",
"expm1": "fail",
"fabs": "pass",
"fdim": "fail",
"floor": "pass",
"fma": "pass",
"fmax": "pass",
"fmin": "pass",
"fmod": "pass",
"fract": "fail",
"frexp": "pass",
"hypot": "fail",
"ilogb": "fail",
"isequal": "pass",
"isfinite": "pass",
"isgreater": "pass",
"isgreaterequal": "pass",
"isinf": "pass",
"isless": "pass",
"islessequal": "pass",
"islessgreater": "pass",
"isnan": "pass",
"isnormal": "pass",
"isnotequal": "pass",
"isordered": "pass",
"isunordered": "pass",
"ldexp": "fail",
"lgamma": "fail",
"lgamma_r": "fail",
"log": "fail",
"log2": "pass",
"log10": "pass",
"log1p": "fail",
"logb": "fail",
"mad": "pass",
"maxmag": "pass",
"minmag": "pass",
"modf": "fail",
"nan": "pass",
"nextafter": "fail",
"pow": "pass",
"pown": "fail",
"powr": "fail",
"remainder": "fail",
"remquo": "fail",
"rint": "pass",
"rootn": "fail",
"round": "pass",
"rsqrt": "pass",
"signbit": "pass",
"sin": "pass",
"sincos": "fail",
"sinh": "pass",
"sinpi": "fail",
"sqrt": "pass",
"sqrt_cr": "pass",
"tan": "pass",
"tanh": "pass",
"tanpi": "fail",
"tgamma": "fail",
"trunc": "pass",
"half_cos": "pass",
"half_divide": "pass",
"half_exp": "pass",
"half_exp2": "pass",
"half_exp10": "pass",
"half_log": "pass",
"half_log2": "pass",
"half_log10": "pass",
"half_powr": "pass",
"half_recip": "pass",
"half_rsqrt": "pass",
"half_sin": "pass",
"half_sqrt": "pass",
"half_tan": "pass",
"add": "pass",
"subtract": "pass",
"negation": "pass",
"reciprocal": "pass",
"divide": "pass",
"divide_cr": "pass",
"multiply": "pass",
"assignment": "pass",
"not": "pass",
"erf": "fail",
"erfc": "fail"
}
},
"test_cl_copy_images": {
"--num-worker-threads 2 small_images": {
"1D": "pass"
}
},
"test_computeinfo": {
"computeinfo": "pass",
"device_uuid": "skip",
"extended_versioning": "pass",
"conformance_version": "pass",
"pci_bus_info": "skip"
"": {
"computeinfo": "pass",
"device_uuid": "skip",
"extended_versioning": "pass",
"conformance_version": "pass",
"pci_bus_info": "skip"
}
},
"test_image_streams": {
"--num-worker-threads 2 CL_FILTER_NEAREST CL_R": {
"1D": "fail"
}
}
}
46 changes: 28 additions & 18 deletions test_common/harness/parseParameters.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,43 +91,38 @@ For spir-v mode only:
"\n");
}

int parseCustomParam(int argc, const char *argv[], const char *ignore)
int parseCommonParamAndGetRemovedArgs(int argc, const char *argv[],
std::vector<std::string> &removed_args,
bool &help)
{
int delArg = 0;
help = false;

for (int i = 1; i < argc; i++)
{
if (ignore != 0)
{
// skip parameters that require special/different treatment in
// application (generic interpretation and parameter removal will
// not be performed)
const char *ptr = strstr(ignore, argv[i]);
if (ptr != 0 && (ptr == ignore || ptr[-1] == ' ')
&& // first on list or ' ' before
(ptr[strlen(argv[i])] == 0
|| ptr[strlen(argv[i])] == ' ')) // last on list or ' ' after
continue;
}

delArg = 0;
size_t i_object_length = strlen("--invalid-object-scenarios=");

if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0)
{
// Note: we don't increment delArg to delete this argument,
// to allow the caller's argument parsing routine to see the
// option and print its own help.
helpInfo();
delArg++;
if (!help)
{
help = true;
removed_args.push_back("--help");
helpInfo();
}
}
else if (!strcmp(argv[i], "--list") || !strcmp(argv[i], "-list"))
{
delArg++;
removed_args.push_back(argv[i]);
gListTests = true;
}
else if (!strcmp(argv[i], "--wimpy") || !strcmp(argv[i], "-w"))
{
delArg++;
removed_args.push_back("--wimpy");
gWimpyMode = true;
}
else if (!strcmp(argv[i], "--compilation-mode"))
Expand Down Expand Up @@ -163,6 +158,7 @@ int parseCustomParam(int argc, const char *argv[], const char *ignore)
" --compilation-mode <online|binary|spir-v>\n");
return -1;
}
removed_args.push_back(std::string(argv[i]) + " " + argv[i + 1]);
}
else if (!strcmp(argv[i], "--num-worker-threads"))
{
Expand All @@ -180,6 +176,7 @@ int parseCustomParam(int argc, const char *argv[], const char *ignore)
"A parameter to --num-worker-threads must be provided!\n");
return -1;
}
removed_args.push_back(std::string(argv[i]) + " " + argv[i + 1]);
}
else if (!strcmp(argv[i], "--compilation-cache-mode"))
{
Expand Down Expand Up @@ -221,6 +218,7 @@ int parseCustomParam(int argc, const char *argv[], const char *ignore)
"<compile-if-absent|force-read|overwrite>\n");
return -1;
}
removed_args.push_back(std::string(argv[i]) + " " + argv[i + 1]);
}
else if (!strcmp(argv[i], "--compilation-cache-path"))
{
Expand All @@ -236,6 +234,7 @@ int parseCustomParam(int argc, const char *argv[], const char *ignore)
"specified.\n");
return -1;
}
removed_args.push_back(std::string(argv[i]) + " " + argv[i + 1]);
}
else if (!strcmp(argv[i], "--compilation-program"))
{
Expand All @@ -251,10 +250,12 @@ int parseCustomParam(int argc, const char *argv[], const char *ignore)
"specified.\n");
return -1;
}
removed_args.push_back(std::string(argv[i]) + " " + argv[i + 1]);
}
else if (!strcmp(argv[i], "--disable-spirv-validation"))
{
delArg++;
removed_args.push_back(argv[i]);
gDisableSPIRVValidation = true;
}
else if (!strcmp(argv[i], "--spirv-validator"))
Expand All @@ -271,6 +272,7 @@ int parseCustomParam(int argc, const char *argv[], const char *ignore)
"specified.\n");
return -1;
}
removed_args.push_back(std::string(argv[i]) + " " + argv[i + 1]);
}
else if (!strncmp(argv[i],
"--invalid-object-scenarios=", i_object_length))
Expand All @@ -297,6 +299,7 @@ int parseCustomParam(int argc, const char *argv[], const char *ignore)
"not specified.\n");
return -1;
}
removed_args.push_back(argv[i]);
}

// cleaning parameters from argv tab
Expand All @@ -317,6 +320,13 @@ int parseCustomParam(int argc, const char *argv[], const char *ignore)
return argc;
}

int parseCommonParam(int argc, const char *argv[])
{
std::vector<std::string> unused1;
bool unused2;
return parseCommonParamAndGetRemovedArgs(argc, argv, unused1, unused2);
}

bool is_power_of_two(int number) { return number && !(number & (number - 1)); }

extern void parseWimpyReductionFactor(const char *&arg,
Expand Down
8 changes: 6 additions & 2 deletions test_common/harness/parseParameters.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

#include "compat.h"
#include <string>
#include <vector>

enum CompilationMode
{
Expand All @@ -43,8 +44,11 @@ extern std::string gSPIRVValidator;
extern bool gListTests;
extern bool gWimpyMode;

extern int parseCustomParam(int argc, const char *argv[],
const char *ignore = 0);
extern int
parseCommonParamAndGetRemovedArgs(int argc, const char *argv[],
std::vector<std::string> &removed_args,
bool &help);
extern int parseCommonParam(int argc, const char *argv[]);

extern void parseWimpyReductionFactor(const char *&arg,
int &wimpyReductionFactor);
Expand Down
Loading
Loading