From 1cf097a53cc30d218a9644e9b020d30904ed0722 Mon Sep 17 00:00:00 2001 From: Bartosz Kostrzewa Date: Thu, 15 Sep 2022 14:15:53 +0000 Subject: [PATCH 1/7] adjust to QudaCABasis -> QudaPolynomialBasis --- quda_dummy_types.h | 4 ++-- quda_types.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/quda_dummy_types.h b/quda_dummy_types.h index b689050dc..7db95bf6a 100644 --- a/quda_dummy_types.h +++ b/quda_dummy_types.h @@ -42,11 +42,11 @@ typedef enum QudaInverterType_s { QUDA_CA_GCR_INVERTER = CA_GCR } QudaInverterType; -typedef enum QudaCABasis_s { +typedef enum QudaPolynomialBasis_s { QUDA_POWER_BASIS, QUDA_CHEBYSHEV_BASIS, QUDA_INVALID_BASIS = QUDA_INVALID_ENUM -} QudaCABasis; +} QudaPolynomialBasis; typedef enum QudaEigSpectrumType_s { QUDA_SPECTRUM_SR_EIG, diff --git a/quda_types.h b/quda_types.h index 009c70925..b0ed3c927 100644 --- a/quda_types.h +++ b/quda_types.h @@ -82,12 +82,12 @@ typedef struct tm_QudaParams_t { // parameters related to communication-avoiding // solvers - QudaCABasis mg_setup_ca_basis[QUDA_MAX_MG_LEVEL]; + QudaPolynomialBasis mg_setup_ca_basis[QUDA_MAX_MG_LEVEL]; int mg_setup_ca_basis_size[QUDA_MAX_MG_LEVEL]; double mg_setup_ca_lambda_min[QUDA_MAX_MG_LEVEL]; double mg_setup_ca_lambda_max[QUDA_MAX_MG_LEVEL]; - QudaCABasis mg_coarse_solver_ca_basis[QUDA_MAX_MG_LEVEL]; + QudaPolynomialBasis mg_coarse_solver_ca_basis[QUDA_MAX_MG_LEVEL]; int mg_coarse_solver_ca_basis_size[QUDA_MAX_MG_LEVEL]; double mg_coarse_solver_ca_lambda_min[QUDA_MAX_MG_LEVEL]; double mg_coarse_solver_ca_lambda_max[QUDA_MAX_MG_LEVEL]; From 0968a7084334111faec684649978e4495741625e Mon Sep 17 00:00:00 2001 From: Bartosz Kostrzewa Date: Thu, 15 Sep 2022 14:20:32 +0000 Subject: [PATCH 2/7] update comment which explains the purpose of tm_QudaParams_t --- quda_types.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/quda_types.h b/quda_types.h index b0ed3c927..7270f69f7 100644 --- a/quda_types.h +++ b/quda_types.h @@ -42,14 +42,16 @@ typedef enum tm_quda_ferm_bc_t { } tm_quda_ferm_bc_t; /* tm_QudaParams_t provides an interface between the tmLQCD input file and the - * available QUDA parameters. At the moment, only the fermionic bounday conditions - * and the MG parameters are exposed like this, but a further refactoring might - * turn this into a complete representation of the possible input parameters */ + * available QUDA parameters for those where we don't + * a) rely on the defaults supplied by the various newQuda****Param() functions + * b) hard-code in quda_interface.c + * c) generate dynamically in quda_interface.c + * */ typedef struct tm_QudaParams_t { int enable_device_memory_pool; int enable_pinned_memory_pool; - tm_quda_ferm_bc_t fermionbc; + tm_quda_ferm_bc_t fermionbc; int pipeline; double reliable_delta; From 06362e745d22b18cdc2d332cc60d49d899aaa876 Mon Sep 17 00:00:00 2001 From: Bartosz Kostrzewa Date: Thu, 15 Sep 2022 14:25:45 +0000 Subject: [PATCH 3/7] some parameters have disappeared from QudaMultigridParam --- quda_interface.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/quda_interface.c b/quda_interface.c index 36c7974ed..8fad0f369 100644 --- a/quda_interface.c +++ b/quda_interface.c @@ -5,6 +5,7 @@ * 2018 Bartosz Kostrzewa, Ferenc Pittler * 2019, 2020 Bartosz Kostrzewa * 2021 Bartosz Kostrzewa, Marco Garofalo, Ferenc Pittler, Simone Bacchio + * 2022 Bartosz Kostrzewa, Marco Garofalo, Simone Romiti * * This file is part of tmLQCD. * @@ -2068,9 +2069,6 @@ void _setQudaMultigridParam(QudaMultigridParam* mg_param) { // only coarsen the spin on the first restriction mg_param->spin_block_size[0] = 2; - mg_param->compute_null_vector = QUDA_COMPUTE_NULL_VECTOR_YES; - mg_param->generate_all_levels = QUDA_BOOLEAN_YES; - mg_param->run_low_mode_check = quda_input.mg_run_low_mode_check; mg_param->run_oblique_proj_check = quda_input.mg_run_oblique_proj_check; mg_param->run_verify = quda_input.mg_run_verify; From ed383432b1cd32f112d0e38783308808a8bd8d7e Mon Sep 17 00:00:00 2001 From: Bartosz Kostrzewa Date: Thu, 15 Sep 2022 14:33:57 +0000 Subject: [PATCH 4/7] forgot to adjust read_input for QudaCABasis -> QudaPolynomialBasis transition --- read_input.l | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/read_input.l b/read_input.l index 738835af7..3bf9a6ca4 100644 --- a/read_input.l +++ b/read_input.l @@ -437,7 +437,7 @@ static inline double fltlist_next_token(int * const list_end){ free(input_copy); } - static inline void parse_quda_mg_cabasis_par_array(char const * const input, QudaCABasis * par_array){ + static inline void parse_quda_mg_polybasis_par_array(char const * const input, QudaPolynomialBasis * par_array){ char paramname[100]; char token[100]; char error_message[ERR_MSG_LEN]; @@ -1947,7 +1947,7 @@ static inline double fltlist_next_token(int * const list_end){ parse_quda_mg_dbl_par_array(yytext, &(quda_input.mg_eig_amax[0])); } {SPC}*MGSetupCABasisType{EQL}{STRLIST} { - parse_quda_mg_cabasis_par_array(yytext, &(quda_input.mg_setup_ca_basis[0])); + parse_quda_mg_polybasis_par_array(yytext, &(quda_input.mg_setup_ca_basis[0])); } {SPC}*MGSetupCABasisSize{EQL}{STRLIST} { parse_quda_mg_int_par_array(yytext, &(quda_input.mg_setup_ca_basis_size[0])); @@ -1959,7 +1959,7 @@ static inline double fltlist_next_token(int * const list_end){ parse_quda_mg_dbl_par_array(yytext, &(quda_input.mg_setup_ca_lambda_max[0])); } {SPC}*MGCoarseSolverCABasisType{EQL}{STRLIST} { - parse_quda_mg_cabasis_par_array(yytext, &(quda_input.mg_coarse_solver_ca_basis[0])); + parse_quda_mg_polybasis_par_array(yytext, &(quda_input.mg_coarse_solver_ca_basis[0])); } {SPC}*MGCoarseSolverCABasisSize{EQL}{STRLIST} { parse_quda_mg_int_par_array(yytext, &(quda_input.mg_coarse_solver_ca_basis_size[0])); From f1d634c73e38f51d96bfe15d50c91cbd9649eec6 Mon Sep 17 00:00:00 2001 From: Bartosz Kostrzewa Date: Thu, 15 Sep 2022 14:50:07 +0000 Subject: [PATCH 5/7] hopefully adjust to all interface-breaking changes from QUDA PR #1283 --- quda_dummy_types.h | 10 ++++ quda_interface.c | 7 +++ quda_types.h | 126 +++++++++++++++++++++++---------------------- 3 files changed, 81 insertions(+), 62 deletions(-) diff --git a/quda_dummy_types.h b/quda_dummy_types.h index 7db95bf6a..db6a995e7 100644 --- a/quda_dummy_types.h +++ b/quda_dummy_types.h @@ -96,4 +96,14 @@ typedef struct QudaInverParam_s { QudaPrecision clover_cuda_prec_eigensolver; } QudaInvertParam; +typedef enum QudaNullVectorSetupType_s { + QUDA_SETUP_NULL_VECTOR_INVERSE_ITERATIONS, + QUDA_SETUP_NULL_VECTOR_CHEBYSHEV_FILTER, + QUDA_SETUP_NULL_VECTOR_EIGENVECTORS, + QUDA_SETUP_NULL_VECTOR_TEST_VECTORS, + QUDA_SETUP_NULL_VECTOR_RESTRICT_FINE, + QUDA_SETUP_NULL_VECTOR_FREE_FIELD, + QUDA_SETUP_NULL_VECTOR_INVALID = QUDA_INVALID_ENUM +} QudaNullVectorSetupType; + #endif diff --git a/quda_interface.c b/quda_interface.c index 8fad0f369..17ea456dc 100644 --- a/quda_interface.c +++ b/quda_interface.c @@ -1947,6 +1947,13 @@ void _setQudaMultigridParam(QudaMultigridParam* mg_param) { mg_param->verbosity[level] = quda_input.mg_verbosity[level]; mg_param->precision_null[level] = QUDA_HALF_PRECISION; + // different setup types exist but inverse iterations are the default + // in the future we might explore also the Chebyshev filter with refinement to + // generate null vectors + mg_param->setup_type[level] = QUDA_SETUP_NULL_VECTOR_INVERSE_ITERATIONS; + mg_param->setup_restrict_remaining_type[level] = QUDA_SETUP_NULL_VECTOR_INVERSE_ITERATIONS; + mg_param->setup_maxiter_inverse_iterations_refinement[level] = 0; + mg_param->setup_inv_type[level] = quda_input.mg_setup_inv_type; // Kate says: experimental, leave at 1 (will be used for bootstrap-style setup later) mg_param->num_setup_iter[level] = 1; diff --git a/quda_types.h b/quda_types.h index 7270f69f7..946cc1619 100644 --- a/quda_types.h +++ b/quda_types.h @@ -48,73 +48,75 @@ typedef enum tm_quda_ferm_bc_t { * c) generate dynamically in quda_interface.c * */ typedef struct tm_QudaParams_t { - int enable_device_memory_pool; - int enable_pinned_memory_pool; - - tm_quda_ferm_bc_t fermionbc; - - int pipeline; - double reliable_delta; - int gcrNkrylov; - - int mg_n_level; - QudaVerbosity mg_verbosity[QUDA_MAX_MG_LEVEL]; - int mg_n_vec[QUDA_MAX_MG_LEVEL]; - int mg_blocksize[QUDA_MAX_MG_LEVEL][4]; - double mg_mu_factor[QUDA_MAX_MG_LEVEL]; - QudaInverterType mg_setup_inv_type; - double mg_setup_2kappamu; - double mg_setup_tol[QUDA_MAX_MG_LEVEL]; - int mg_setup_maxiter[QUDA_MAX_MG_LEVEL]; - QudaInverterType mg_coarse_solver_type[QUDA_MAX_MG_LEVEL]; - int mg_coarse_solver_maxiter[QUDA_MAX_MG_LEVEL]; - double mg_coarse_solver_tol[QUDA_MAX_MG_LEVEL]; - int mg_nu_pre[QUDA_MAX_MG_LEVEL]; - int mg_nu_post[QUDA_MAX_MG_LEVEL]; - QudaInverterType mg_smoother_type[QUDA_MAX_MG_LEVEL]; - double mg_smoother_tol[QUDA_MAX_MG_LEVEL]; - double mg_omega[QUDA_MAX_MG_LEVEL]; - int mg_run_verify; - int mg_enable_size_three_blocks; - double mg_reuse_setup_mu_threshold; - double mg_reset_setup_mdu_threshold; - double mg_refresh_setup_mdu_threshold; - - int mg_setup_maxiter_refresh[QUDA_MAX_MG_LEVEL]; - + int enable_device_memory_pool; + int enable_pinned_memory_pool; + + tm_quda_ferm_bc_t fermionbc; + + int pipeline; + double reliable_delta; + int gcrNkrylov; + + int mg_n_level; + QudaVerbosity mg_verbosity[QUDA_MAX_MG_LEVEL]; + int mg_n_vec[QUDA_MAX_MG_LEVEL]; + int mg_blocksize[QUDA_MAX_MG_LEVEL][4]; + double mg_mu_factor[QUDA_MAX_MG_LEVEL]; + QudaInverterType mg_setup_inv_type; + double mg_setup_2kappamu; + double mg_setup_tol[QUDA_MAX_MG_LEVEL]; + int mg_setup_maxiter[QUDA_MAX_MG_LEVEL]; + QudaInverterType mg_coarse_solver_type[QUDA_MAX_MG_LEVEL]; + int mg_coarse_solver_maxiter[QUDA_MAX_MG_LEVEL]; + double mg_coarse_solver_tol[QUDA_MAX_MG_LEVEL]; + int mg_nu_pre[QUDA_MAX_MG_LEVEL]; + int mg_nu_post[QUDA_MAX_MG_LEVEL]; + QudaInverterType mg_smoother_type[QUDA_MAX_MG_LEVEL]; + double mg_smoother_tol[QUDA_MAX_MG_LEVEL]; + double mg_omega[QUDA_MAX_MG_LEVEL]; + int mg_run_verify; + int mg_enable_size_three_blocks; + double mg_reuse_setup_mu_threshold; + double mg_reset_setup_mdu_threshold; + double mg_refresh_setup_mdu_threshold; + + int mg_setup_maxiter_refresh[QUDA_MAX_MG_LEVEL]; + + QudaNullVectorSetupType mg_setup_type[QUDA_MG_MAX_LEVEL]; + // parameters related to communication-avoiding // solvers - QudaPolynomialBasis mg_setup_ca_basis[QUDA_MAX_MG_LEVEL]; - int mg_setup_ca_basis_size[QUDA_MAX_MG_LEVEL]; - double mg_setup_ca_lambda_min[QUDA_MAX_MG_LEVEL]; - double mg_setup_ca_lambda_max[QUDA_MAX_MG_LEVEL]; + QudaPolynomialBasis mg_setup_ca_basis[QUDA_MAX_MG_LEVEL]; + int mg_setup_ca_basis_size[QUDA_MAX_MG_LEVEL]; + double mg_setup_ca_lambda_min[QUDA_MAX_MG_LEVEL]; + double mg_setup_ca_lambda_max[QUDA_MAX_MG_LEVEL]; - QudaPolynomialBasis mg_coarse_solver_ca_basis[QUDA_MAX_MG_LEVEL]; - int mg_coarse_solver_ca_basis_size[QUDA_MAX_MG_LEVEL]; - double mg_coarse_solver_ca_lambda_min[QUDA_MAX_MG_LEVEL]; - double mg_coarse_solver_ca_lambda_max[QUDA_MAX_MG_LEVEL]; + QudaPolynomialBasis mg_coarse_solver_ca_basis[QUDA_MAX_MG_LEVEL]; + int mg_coarse_solver_ca_basis_size[QUDA_MAX_MG_LEVEL]; + double mg_coarse_solver_ca_lambda_min[QUDA_MAX_MG_LEVEL]; + double mg_coarse_solver_ca_lambda_max[QUDA_MAX_MG_LEVEL]; // parameters related to coarse grid deflation in the MG - int mg_use_eig_solver[QUDA_MAX_MG_LEVEL]; - int mg_eig_preserve_deflation; - int mg_eig_nEv[QUDA_MAX_MG_LEVEL]; - int mg_eig_nKr[QUDA_MAX_MG_LEVEL]; - int mg_eig_require_convergence[QUDA_MAX_MG_LEVEL]; - int mg_eig_check_interval[QUDA_MAX_MG_LEVEL]; - int mg_eig_max_restarts[QUDA_MAX_MG_LEVEL]; - double mg_eig_tol[QUDA_MAX_MG_LEVEL]; - int mg_eig_use_poly_acc[QUDA_MAX_MG_LEVEL]; - int mg_eig_poly_deg[QUDA_MAX_MG_LEVEL]; - double mg_eig_amin[QUDA_MAX_MG_LEVEL]; - double mg_eig_amax[QUDA_MAX_MG_LEVEL]; - int mg_eig_use_normop[QUDA_MAX_MG_LEVEL]; - int mg_eig_use_dagger[QUDA_MAX_MG_LEVEL]; - QudaEigSpectrumType mg_eig_spectrum[QUDA_MAX_MG_LEVEL]; - QudaEigType mg_eig_type[QUDA_MAX_MG_LEVEL]; - int mg_coarse_guess; - - int mg_run_low_mode_check; - int mg_run_oblique_proj_check; + int mg_use_eig_solver[QUDA_MAX_MG_LEVEL]; + int mg_eig_preserve_deflation; + int mg_eig_nEv[QUDA_MAX_MG_LEVEL]; + int mg_eig_nKr[QUDA_MAX_MG_LEVEL]; + int mg_eig_require_convergence[QUDA_MAX_MG_LEVEL]; + int mg_eig_check_interval[QUDA_MAX_MG_LEVEL]; + int mg_eig_max_restarts[QUDA_MAX_MG_LEVEL]; + double mg_eig_tol[QUDA_MAX_MG_LEVEL]; + int mg_eig_use_poly_acc[QUDA_MAX_MG_LEVEL]; + int mg_eig_poly_deg[QUDA_MAX_MG_LEVEL]; + double mg_eig_amin[QUDA_MAX_MG_LEVEL]; + double mg_eig_amax[QUDA_MAX_MG_LEVEL]; + int mg_eig_use_normop[QUDA_MAX_MG_LEVEL]; + int mg_eig_use_dagger[QUDA_MAX_MG_LEVEL]; + QudaEigSpectrumType mg_eig_spectrum[QUDA_MAX_MG_LEVEL]; + QudaEigType mg_eig_type[QUDA_MAX_MG_LEVEL]; + int mg_coarse_guess; + + int mg_run_low_mode_check; + int mg_run_oblique_proj_check; } tm_QudaParams_t; From 125baef0e6bfcea5d3e20f8a521439038a3afea5 Mon Sep 17 00:00:00 2001 From: Bartosz Kostrzewa Date: Thu, 15 Sep 2022 15:28:24 +0000 Subject: [PATCH 6/7] fix typo --- quda_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quda_types.h b/quda_types.h index 946cc1619..cace8fcaa 100644 --- a/quda_types.h +++ b/quda_types.h @@ -82,7 +82,7 @@ typedef struct tm_QudaParams_t { int mg_setup_maxiter_refresh[QUDA_MAX_MG_LEVEL]; - QudaNullVectorSetupType mg_setup_type[QUDA_MG_MAX_LEVEL]; + QudaNullVectorSetupType mg_setup_type[QUDA_MAX_MG_LEVEL]; // parameters related to communication-avoiding // solvers From 82b5e0f74b7782f4245291dba8cdf3aeb300521b Mon Sep 17 00:00:00 2001 From: Bartosz Kostrzewa Date: Mon, 19 Sep 2022 15:49:43 +0000 Subject: [PATCH 7/7] set per-level null-vector I/O parameters for the MG (bug since June 2019, ouch...) --- quda_interface.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/quda_interface.c b/quda_interface.c index 17ea456dc..2b195cbe9 100644 --- a/quda_interface.c +++ b/quda_interface.c @@ -1945,6 +1945,12 @@ void _setQudaMultigridParam(QudaMultigridParam* mg_param) { } // for( dim=0 to dim=3 ) (space-time dimensions) + // set file i/o parameters to a default which imples that no null-vector I/O is performed + strcpy(mg_param->vec_infile[level], ""); + strcpy(mg_param->vec_outfile[level], ""); + mg_param->vec_store[level] = QUDA_BOOLEAN_NO; + mg_param->vec_load[level] = QUDA_BOOLEAN_NO; + mg_param->verbosity[level] = quda_input.mg_verbosity[level]; mg_param->precision_null[level] = QUDA_HALF_PRECISION; // different setup types exist but inverse iterations are the default @@ -2080,9 +2086,6 @@ void _setQudaMultigridParam(QudaMultigridParam* mg_param) { mg_param->run_oblique_proj_check = quda_input.mg_run_oblique_proj_check; mg_param->run_verify = quda_input.mg_run_verify; - // set file i/o parameters - strcpy(mg_param->vec_infile, ""); - strcpy(mg_param->vec_outfile, ""); } int invert_eo_degenerate_quda(spinor * const out,