From ad91e6ec77bd56b63e2da21072b1b9b80d589b13 Mon Sep 17 00:00:00 2001 From: Andrzej Date: Wed, 20 Jan 2021 14:12:27 +0100 Subject: [PATCH] feat: switch to py3 --- .../get_last_by_user_example.py | 2 +- .../example_wrapp_usage/request_wrapper.py | 4 ++-- scripts/example_wrapp_usage/xsdbInsert.py | 14 ++++++------- scripts/example_wrapp_usage/xsdbSearch.py | 4 ++-- scripts/wrapper/request_wrapper.py | 4 ++-- scripts/xsdb_insert.py | 2 +- scripts/xsdb_update.py | 2 +- .../2_retrieve_files_xsec_database.py | 10 +++++----- .../xsec_calc_auto/3_compute_xsec_database.py | 10 +++++----- .../4_prepare_json_xsec_database.py | 14 ++++++------- .../xsec_calc_auto/5_fill_xsec_database.py | 10 +++++----- server/api.py | 20 +++++++++---------- server/config.py | 4 ++-- server/decorators.py | 2 +- server/mailing.py | 6 +++--- server/utils.py | 12 +++++------ server/validate.py | 8 ++++---- 17 files changed, 64 insertions(+), 64 deletions(-) diff --git a/scripts/example_wrapp_usage/get_last_by_user_example.py b/scripts/example_wrapp_usage/get_last_by_user_example.py index 4843e94..b9a41ef 100755 --- a/scripts/example_wrapp_usage/get_last_by_user_example.py +++ b/scripts/example_wrapp_usage/get_last_by_user_example.py @@ -5,4 +5,4 @@ user_name = 'cmsxsec' record = xsdb_req.get_last_inserted_by_user(user_name) -print 'Last record by user [', user_name, '] inserted at [' ,record['createdOn'], ']' +print('Last record by user [', user_name, '] inserted at [' ,record['createdOn'], ']') diff --git a/scripts/example_wrapp_usage/request_wrapper.py b/scripts/example_wrapp_usage/request_wrapper.py index baff75c..fed167d 100755 --- a/scripts/example_wrapp_usage/request_wrapper.py +++ b/scripts/example_wrapp_usage/request_wrapper.py @@ -1,8 +1,8 @@ import subprocess import pycurl import json -from StringIO import StringIO -from urllib import urlencode +from io import StringIO +from urllib.parse import urlencode import os class RequestWrapper: diff --git a/scripts/example_wrapp_usage/xsdbInsert.py b/scripts/example_wrapp_usage/xsdbInsert.py index 310168b..688a1b4 100755 --- a/scripts/example_wrapp_usage/xsdbInsert.py +++ b/scripts/example_wrapp_usage/xsdbInsert.py @@ -2,13 +2,13 @@ xsdb_request = RequestWrapper() -crossSection = raw_input("Enter the Cross Section!!!") -totalUncertainty = raw_input("Enter the total Uncertainty!!!") -shower = raw_input("Enter the hadronization (shower) tool!!!") -das = raw_input("Enter the DAS primary Dataset name!!!") -mcm = raw_input("Enter the MCM prePID!!!") -accuracy = raw_input("Enter the sample accuracy!!!") -energy = raw_input("Enter the beam energy!!!") +crossSection = input("Enter the Cross Section!!!") +totalUncertainty = input("Enter the total Uncertainty!!!") +shower = input("Enter the hadronization (shower) tool!!!") +das = input("Enter the DAS primary Dataset name!!!") +mcm = input("Enter the MCM prePID!!!") +accuracy = input("Enter the sample accuracy!!!") +energy = input("Enter the beam energy!!!") query = { "cross_section": crossSection, diff --git a/scripts/example_wrapp_usage/xsdbSearch.py b/scripts/example_wrapp_usage/xsdbSearch.py index 3567efb..cd384fb 100755 --- a/scripts/example_wrapp_usage/xsdbSearch.py +++ b/scripts/example_wrapp_usage/xsdbSearch.py @@ -2,8 +2,8 @@ xsdb_request = RequestWrapper() -key = raw_input("Enter the search key that you want!!!") -value = raw_input("Enter the search value that you would like to query!!!") +key = input("Enter the search key that you want!!!") +value = input("Enter the search value that you would like to query!!!") query = {} query[key] = value diff --git a/scripts/wrapper/request_wrapper.py b/scripts/wrapper/request_wrapper.py index baff75c..fed167d 100755 --- a/scripts/wrapper/request_wrapper.py +++ b/scripts/wrapper/request_wrapper.py @@ -1,8 +1,8 @@ import subprocess import pycurl import json -from StringIO import StringIO -from urllib import urlencode +from io import StringIO +from urllib.parse import urlencode import os class RequestWrapper: diff --git a/scripts/xsdb_insert.py b/scripts/xsdb_insert.py index 3480fce..6cf39ce 100755 --- a/scripts/xsdb_insert.py +++ b/scripts/xsdb_insert.py @@ -64,6 +64,6 @@ args = parser.parse_args() args_dict = vars(args) -request_dict = dict((key, value) for key, value in args_dict.iteritems() if value is not None) +request_dict = dict((key, value) for key, value in args_dict.items() if value is not None) xsdb_req.insert(keyval_dict=request_dict) \ No newline at end of file diff --git a/scripts/xsdb_update.py b/scripts/xsdb_update.py index b81afe8..eca0325 100755 --- a/scripts/xsdb_update.py +++ b/scripts/xsdb_update.py @@ -67,7 +67,7 @@ args = parser.parse_args() args_dict = vars(args) -request_dict = dict((key, value) for key, value in args_dict.iteritems() if value is not None) +request_dict = dict((key, value) for key, value in args_dict.items() if value is not None) record_id = request_dict['id'] del request_dict['id'] diff --git a/scripts/xsec_calc_auto/2_retrieve_files_xsec_database.py b/scripts/xsec_calc_auto/2_retrieve_files_xsec_database.py index 3e8104d..fd92a9f 100755 --- a/scripts/xsec_calc_auto/2_retrieve_files_xsec_database.py +++ b/scripts/xsec_calc_auto/2_retrieve_files_xsec_database.py @@ -15,9 +15,9 @@ with open('datasets.txt') as f: for dataset in f: dataset = dataset.rstrip('\n') - print dataset.split('/')[1] + print(dataset.split('/')[1]) if not os.path.isfile("getXsec/getXsec_"+dataset.split('/')[1]+".sh"): - print "Creating file" + print("Creating file") os.popen("sh getfiles/getfiles_"+dataset.split('/')[1]+".sh 2>&1 > getXsec/getXsec_"+dataset.split('/')[1]+".sh").read() else: with open("getXsec/getXsec_"+dataset.split('/')[1]+".sh", 'r+') as f: @@ -27,11 +27,11 @@ content2 = "" content2 = f2.read() if not '--skipexisting "False"' in content2: - print "File corrupted, recreating file" + print("File corrupted, recreating file") content2 = content2.replace('--skipexisting "True"','--skipexisting "False"') with open("getfiles/getfiles_"+dataset.split('/')[1]+".sh", 'w') as f2: f2.write(content2) - print "setting skipexisting to False" + print("setting skipexisting to False") os.popen("sh getfiles/getfiles_"+dataset.split('/')[1]+".sh 2>&1 > getXsec/getXsec_"+dataset.split('/')[1]+".sh").read() with open("getfiles/getfiles_"+dataset.split('/')[1]+".sh", 'r') as f2: content2 = "" @@ -41,7 +41,7 @@ with open("getfiles/getfiles_"+dataset.split('/')[1]+".sh", 'w') as f2: f2.write(content2) else: - print "File found" + print("File found") os.system("chmod 755 getXsec/getXsec_"+dataset.split('/')[1]+".sh") # f1=open("xsec_"+dataset.split('/')[1]+"_getfiles.sh", 'w') diff --git a/scripts/xsec_calc_auto/3_compute_xsec_database.py b/scripts/xsec_calc_auto/3_compute_xsec_database.py index c465809..a1c2b42 100755 --- a/scripts/xsec_calc_auto/3_compute_xsec_database.py +++ b/scripts/xsec_calc_auto/3_compute_xsec_database.py @@ -21,25 +21,25 @@ with open('datasets.txt') as f: for dataset in f: dataset = dataset.rstrip('\n') - print dataset.split('/')[1] + print(dataset.split('/')[1]) if not os.path.isfile("getXsec/getXsec_"+dataset.split('/')[1]+".sh"): - print "Input file not found" + print("Input file not found") else: if os.path.isfile("xsec/xsec_"+dataset.split('/')[1]+".log"): with open("xsec/xsec_"+dataset.split('/')[1]+".log", 'r+') as f: content = f.read() if 'final cross section' in content: - print "Cross section already computed, skipping" + print("Cross section already computed, skipping") continue with open("getXsec/getXsec_"+dataset.split('/')[1]+".sh", 'r+') as f: content = f.read() if not '/store' in content: - print "Input script corrupted, skipping" + print("Input script corrupted, skipping") continue if not 'cvmfs' in content: f.seek(0, 0) f.write("cd /cvmfs/cms.cern.ch/"+scram_arch+"/cms/cmssw/"+cmssw+"/\n eval `scramv1 runtime -sh`\n cd "+os.getcwd()+"\n\n" + content) - print "Computing cross section" + print("Computing cross section") if queue != "": os.system("chmod 755 "+os.getcwd()+"/getXsec/getXsec_"+dataset.split('/')[1]+".sh; bsub -q "+queue+" -u ciaociao1 -C 0 "+os.getcwd()+"/getXsec/getXsec_"+dataset.split('/')[1]+".sh; sleep 2") njob = njob+1 diff --git a/scripts/xsec_calc_auto/4_prepare_json_xsec_database.py b/scripts/xsec_calc_auto/4_prepare_json_xsec_database.py index 369bac2..853093b 100755 --- a/scripts/xsec_calc_auto/4_prepare_json_xsec_database.py +++ b/scripts/xsec_calc_auto/4_prepare_json_xsec_database.py @@ -20,15 +20,15 @@ for dataset in f: dataset = dataset.rstrip('\n') primary_dataset_name = dataset.split("/")[1] - print primary_dataset_name + print(primary_dataset_name) if not os.path.isfile("xsec/xsec_"+primary_dataset_name+".log"): - print "Input file not found" + print("Input file not found") else: if not os.path.isfile(json_output_folder+"/xsec_"+primary_dataset_name+".json") or overwrite: with open("xsec/xsec_"+dataset.split('/')[1]+".log", 'r+') as f: content = f.read() if not 'final cross section' in content: - print "Cross section computation not correctly performed, skipping" + print("Cross section computation not correctly performed, skipping") continue if os.path.isfile(json_output_folder+"/xsec_"+primary_dataset_name+".json"): with open(json_output_folder+"/xsec_"+dataset.split('/')[1]+".json", 'r') as f: @@ -41,11 +41,11 @@ and (not "\"MCM\": \"\"," in content) \ and (not "\"MCM\": \"None\"," in content): # and (not "CITo" in primary_dataset_name) \ - print "json OK, skipping" + print("json OK, skipping") continue else: - print "json corrupted, reproducing" - print content + print("json corrupted, reproducing") + print(content) os.system("cp test_xsecdb_insert.json "+json_output_folder+"/xsec_"+primary_dataset_name+".json") os.system("sed -i -e 's/REPLACE_PRIMARY_DATASET_NAME/"+primary_dataset_name+"/g' "+json_output_folder+"/xsec_"+primary_dataset_name+".json") os.system("sed -i -e 's/REPLACE_FULL_DATASET_NAME/"+dataset.replace("/","\/")+"/g' "+json_output_folder+"/xsec_"+primary_dataset_name+".json") @@ -102,7 +102,7 @@ os.system("sed -i -e 's/REPLACE_MCM_PREPID/"+str(mcm_prepid)+"/g' "+json_output_folder+"/xsec_"+primary_dataset_name+".json") with open(json_output_folder+"/xsec_"+dataset.split('/')[1]+".json", 'r+') as f: - print "produced json:\n", f.read() + print("produced json:\n", f.read()) # sys.exit() diff --git a/scripts/xsec_calc_auto/5_fill_xsec_database.py b/scripts/xsec_calc_auto/5_fill_xsec_database.py index 696036f..b684fb4 100755 --- a/scripts/xsec_calc_auto/5_fill_xsec_database.py +++ b/scripts/xsec_calc_auto/5_fill_xsec_database.py @@ -11,28 +11,28 @@ if not filename.endswith(".json"): continue primary_dataset_name = filename.split("xsec_")[1].replace(".json","") - print primary_dataset_name + print(primary_dataset_name) # print filename if os.path.isfile(xsecdb_folder+"/update_logs/filldb_"+primary_dataset_name+".log"): with open(xsecdb_folder+"/update_logs/filldb_"+primary_dataset_name+".log", 'r') as f: content = f.read() if '{"status": "new"' in content and campaign in content: - print 'filldb report existing and valid, skipping dataset' + print('filldb report existing and valid, skipping dataset') continue if os.path.isfile(xsecdb_folder+"/update_logs/filldb_"+primary_dataset_name+"_"+campaign+".log"): with open(xsecdb_folder+"/update_logs/filldb_"+primary_dataset_name+"_"+campaign+".log", 'r') as f: content = f.read() if '{"status": "new"' in content and campaign in content: - print 'filldb report existing and valid, skipping dataset' + print('filldb report existing and valid, skipping dataset') continue with open(json_output_folder+filename, 'r') as f: content = f.read() if not 'REPLACE_' in content: - print "json OK, uploading" + print("json OK, uploading") else: - print "json corrupted, skipping" + print("json corrupted, skipping") continue os.system(" cd "+xsecdb_folder+"/wrapper;\ diff --git a/server/api.py b/server/api.py index 61e81b6..5a9445c 100755 --- a/server/api.py +++ b/server/api.py @@ -1,7 +1,7 @@ import json import re import copy -import logger +from . import logger from flask import Flask, request, jsonify, make_response, render_template from pymongo import MongoClient @@ -10,13 +10,13 @@ from time import gmtime, strftime from flask_cors import CORS -from fields import fields as record_structure -from mailing import send_mail, send_mail_approve -from utils import compile_regex, get_ordered_field_list,\ +from .fields import fields as record_structure +from .mailing import send_mail, send_mail_approve +from .utils import compile_regex, get_ordered_field_list,\ get_user_groups, is_user_in_group, get_field_order, remove_readonly_fields -from validate import validate_model, validate_model_update -from decorators import auth_user_group -from config import CONFIG +from .validate import validate_model, validate_model_update +from .decorators import auth_user_group +from .config import CONFIG app = Flask(__name__, static_folder="../client/dist", template_folder="../client/templates") @@ -56,7 +56,7 @@ def get_by_id(record_id): structure = copy.deepcopy(record_structure) # Map record fields to information in record_structure (type, disabled, required, order) - for key, value in record.iteritems(): + for key, value in record.items(): if key in structure: result_dic[key] = structure[key] # type, disabled, title # overwrite value with true value from db @@ -216,7 +216,7 @@ def search(): @app.route('/api/fields', methods=['GET']) def get_fields(): """ get list of record_structure field names (for selecting visible columns) """ - result = sorted(record_structure.keys(), key=get_field_order) + result = sorted(list(record_structure.keys()), key=get_field_order) return make_response(jsonify(result), 200) @@ -229,7 +229,7 @@ def approve_records(): logger.debug("APPROVE:" + str(record_ids) + " - USER " + user_login) curr_date = strftime("%Y-%m-%d %H:%M:%S", gmtime()) - object_ids = map(lambda x: ObjectId(x), record_ids) + object_ids = [ObjectId(x) for x in record_ids] collection.update_many({'_id': {'$in': object_ids}}, { '$set': { 'status': 'approved', diff --git a/server/config.py b/server/config.py index b892d77..dc3336c 100755 --- a/server/config.py +++ b/server/config.py @@ -1,4 +1,4 @@ -import ConfigParser +import configparser import os import json @@ -9,7 +9,7 @@ def _update(self, **entries): app_dir = os.path.dirname(__file__) conf_dict = {} -cfg_parser = ConfigParser.RawConfigParser() +cfg_parser = configparser.RawConfigParser() a = cfg_parser.read(app_dir + "/config.cfg") for option in cfg_parser.options("strings"): diff --git a/server/decorators.py b/server/decorators.py index c8a4aa5..4e38184 100755 --- a/server/decorators.py +++ b/server/decorators.py @@ -1,6 +1,6 @@ from functools import wraps from flask import request, make_response, jsonify -from utils import get_user_groups, is_user_in_group +from .utils import get_user_groups, is_user_in_group # Decorator wrapped into function which accepts required e-group level def auth_user_group(group_level): diff --git a/server/mailing.py b/server/mailing.py index 8778be9..27cc92f 100755 --- a/server/mailing.py +++ b/server/mailing.py @@ -1,10 +1,10 @@ import smtplib -import logger +from . import logger from email.MIMEMultipart import MIMEMultipart from email.MIMEText import MIMEText from email.Utils import COMMASPACE, formatdate -from config import CONFIG +from .config import CONFIG def send_mail(body, subject, recipients, **kwargs): sender = CONFIG.MAIL_SEND_FROM @@ -22,7 +22,7 @@ def send_mail(body, subject, recipients, **kwargs): smtpObj.sendmail(sender, recipients, msg.as_string()) smtpObj.close() except Exception as e: - print "Error: unable to send email", e.__class__, e.message + print("Error: unable to send email", e.__class__, e.message) def send_mail_approve(record_id): diff --git a/server/utils.py b/server/utils.py index 893a147..3176e38 100755 --- a/server/utils.py +++ b/server/utils.py @@ -1,8 +1,8 @@ import re -import logger +from . import logger from flask import request -from config import CONFIG -from fields import fields as record_structure +from .config import CONFIG +from .fields import fields as record_structure def get_user_groups(): @@ -38,7 +38,7 @@ def compile_regex(in_dic): ''' compile value strings in mongo search query ''' - for key, value in in_dic.iteritems(): + for key, value in in_dic.items(): if key != "$or" and key != "$and": in_dic[key] = re.compile(value, re.I) else: @@ -63,7 +63,7 @@ def get_ordered_field_list(record_dict): transform record_structure dictionary into ordered list ''' result = [] - result_ = sorted(record_dict.iteritems(), key=lambda x: get_field_order(x[0])) + result_ = sorted(iter(record_dict.items()), key=lambda x: get_field_order(x[0])) for tupl in result_: dic = tupl[1] dic['name'] = tupl[0] @@ -76,7 +76,7 @@ def remove_readonly_fields(record_request): remove read only fields from users request ''' - read_only_fields = [key for key, value in record_structure.iteritems() if value.get('read_only', False)] + read_only_fields = [key for key, value in record_structure.items() if value.get('read_only', False)] for field_name in read_only_fields: record_request.pop(field_name, None) diff --git a/server/validate.py b/server/validate.py index d46effc..4278e9f 100755 --- a/server/validate.py +++ b/server/validate.py @@ -1,6 +1,6 @@ -import logger +from . import logger import re -from fields import fields as record_structure +from .fields import fields as record_structure re_whitespace = re.compile(r'\s+') @@ -11,7 +11,7 @@ def validate_model(record): error_obj = {} # validate against required fields from record structure - for key, field in record_structure.iteritems(): + for key, field in record_structure.items(): validate_field_(field, key=key, value=record.get(key), error_obj=error_obj) logger.debug(error_obj) @@ -25,7 +25,7 @@ def validate_model_update(record_part): ''' error_obj = {} - for key, value in record_part.iteritems(): + for key, value in record_part.items(): field = record_structure.get(key) validate_field_(field, key, value, error_obj)