feat: switch to py3

scripts/example_wrapp_usage/get_last_by_user_example.py

@@ -5,4 +5,4 @@
 user_name = 'cmsxsec'
 record = xsdb_req.get_last_inserted_by_user(user_name)
 
-print 'Last record by user [', user_name, '] inserted at [' ,record['createdOn'], ']'
+print('Last record by user [', user_name, '] inserted at [' ,record['createdOn'], ']')

scripts/example_wrapp_usage/request_wrapper.py

@@ -1,8 +1,8 @@
 import subprocess
 import pycurl
 import json
-from StringIO import StringIO 
-from urllib import urlencode
+from io import StringIO 
+from urllib.parse import urlencode
 import os
 
 class RequestWrapper:

scripts/example_wrapp_usage/xsdbInsert.py

@@ -2,13 +2,13 @@
 
 xsdb_request = RequestWrapper()
 
-crossSection = raw_input("Enter the Cross Section!!!")
-totalUncertainty = raw_input("Enter the total Uncertainty!!!")
-shower = raw_input("Enter the hadronization (shower) tool!!!")
-das = raw_input("Enter the DAS primary Dataset name!!!")
-mcm = raw_input("Enter the MCM prePID!!!")
-accuracy = raw_input("Enter the sample accuracy!!!")
-energy = raw_input("Enter the beam energy!!!")
+crossSection = input("Enter the Cross Section!!!")
+totalUncertainty = input("Enter the total Uncertainty!!!")
+shower = input("Enter the hadronization (shower) tool!!!")
+das = input("Enter the DAS primary Dataset name!!!")
+mcm = input("Enter the MCM prePID!!!")
+accuracy = input("Enter the sample accuracy!!!")
+energy = input("Enter the beam energy!!!")
 
 query = {
     "cross_section": crossSection,

scripts/example_wrapp_usage/xsdbSearch.py

@@ -2,8 +2,8 @@
 
 xsdb_request = RequestWrapper()
 
-key = raw_input("Enter the search key that you want!!!")
-value = raw_input("Enter the search value that you would like to query!!!")
+key = input("Enter the search key that you want!!!")
+value = input("Enter the search value that you would like to query!!!")
 
 query = {}
 query[key] = value

scripts/wrapper/request_wrapper.py

@@ -1,8 +1,8 @@
 import subprocess
 import pycurl
 import json
-from StringIO import StringIO 
-from urllib import urlencode
+from io import StringIO 
+from urllib.parse import urlencode
 import os
 
 class RequestWrapper:

scripts/xsdb_insert.py

@@ -64,6 +64,6 @@
 
 args = parser.parse_args()
 args_dict = vars(args)
-request_dict = dict((key, value) for key, value in args_dict.iteritems() if value is not None)
+request_dict = dict((key, value) for key, value in args_dict.items() if value is not None)
 
 xsdb_req.insert(keyval_dict=request_dict)

scripts/xsdb_update.py

@@ -67,7 +67,7 @@
 
 args = parser.parse_args()
 args_dict = vars(args)
-request_dict = dict((key, value) for key, value in args_dict.iteritems() if value is not None)
+request_dict = dict((key, value) for key, value in args_dict.items() if value is not None)
 
 record_id = request_dict['id']
 del request_dict['id']

scripts/xsec_calc_auto/2_retrieve_files_xsec_database.py

@@ -15,9 +15,9 @@
 with open('datasets.txt') as f:
     for dataset in f:
         dataset = dataset.rstrip('\n')
-        print dataset.split('/')[1]
+        print(dataset.split('/')[1])
         if not os.path.isfile("getXsec/getXsec_"+dataset.split('/')[1]+".sh"):
-            print "Creating file"
+            print("Creating file")
             os.popen("sh getfiles/getfiles_"+dataset.split('/')[1]+".sh 2>&1 > getXsec/getXsec_"+dataset.split('/')[1]+".sh").read()
         else:
             with open("getXsec/getXsec_"+dataset.split('/')[1]+".sh", 'r+') as f:
@@ -27,11 +27,11 @@
                         content2 = ""
                         content2 = f2.read()
                         if not '--skipexisting "False"' in content2:
-                            print "File corrupted, recreating file"
+                            print("File corrupted, recreating file")
                             content2 = content2.replace('--skipexisting "True"','--skipexisting "False"')
                             with open("getfiles/getfiles_"+dataset.split('/')[1]+".sh", 'w') as f2:
                                 f2.write(content2)
-                                print "setting skipexisting to False"
+                                print("setting skipexisting to False")
                     os.popen("sh getfiles/getfiles_"+dataset.split('/')[1]+".sh 2>&1 > getXsec/getXsec_"+dataset.split('/')[1]+".sh").read()
                     with open("getfiles/getfiles_"+dataset.split('/')[1]+".sh", 'r') as f2:
                         content2 = ""
@@ -41,7 +41,7 @@
                             with open("getfiles/getfiles_"+dataset.split('/')[1]+".sh", 'w') as f2:
                                 f2.write(content2)
                 else:
-                    print "File found"
+                    print("File found")
         os.system("chmod 755 getXsec/getXsec_"+dataset.split('/')[1]+".sh")
 
         # f1=open("xsec_"+dataset.split('/')[1]+"_getfiles.sh", 'w')

scripts/xsec_calc_auto/3_compute_xsec_database.py

@@ -21,25 +21,25 @@
 with open('datasets.txt') as f:
     for dataset in f:
         dataset = dataset.rstrip('\n')
-        print dataset.split('/')[1]
+        print(dataset.split('/')[1])
         if not os.path.isfile("getXsec/getXsec_"+dataset.split('/')[1]+".sh"):
-            print "Input file not found"
+            print("Input file not found")
         else:
             if os.path.isfile("xsec/xsec_"+dataset.split('/')[1]+".log"):
                 with open("xsec/xsec_"+dataset.split('/')[1]+".log", 'r+') as f:
                     content = f.read()
                     if 'final cross section' in content: 
-                        print "Cross section already computed, skipping"
+                        print("Cross section already computed, skipping")
                         continue
             with open("getXsec/getXsec_"+dataset.split('/')[1]+".sh", 'r+') as f:
                 content = f.read()
                 if not '/store' in content:
-                    print "Input script corrupted, skipping"
+                    print("Input script corrupted, skipping")
                     continue
                 if not 'cvmfs' in content:
                     f.seek(0, 0)
                     f.write("cd /cvmfs/cms.cern.ch/"+scram_arch+"/cms/cmssw/"+cmssw+"/\n eval `scramv1 runtime -sh`\n cd "+os.getcwd()+"\n\n" + content)
-            print "Computing cross section"
+            print("Computing cross section")
             if queue != "":
                 os.system("chmod 755 "+os.getcwd()+"/getXsec/getXsec_"+dataset.split('/')[1]+".sh; bsub -q "+queue+" -u ciaociao1 -C 0 "+os.getcwd()+"/getXsec/getXsec_"+dataset.split('/')[1]+".sh; sleep 2")
                 njob = njob+1

scripts/xsec_calc_auto/4_prepare_json_xsec_database.py

@@ -20,15 +20,15 @@
     for dataset in f:
         dataset = dataset.rstrip('\n')
         primary_dataset_name = dataset.split("/")[1]
-        print primary_dataset_name
+        print(primary_dataset_name)
         if not os.path.isfile("xsec/xsec_"+primary_dataset_name+".log"):
-            print "Input file not found"
+            print("Input file not found")
         else:
             if not os.path.isfile(json_output_folder+"/xsec_"+primary_dataset_name+".json") or overwrite:
                 with open("xsec/xsec_"+dataset.split('/')[1]+".log", 'r+') as f:
                     content = f.read()
                     if not 'final cross section' in content: 
-                        print "Cross section computation not correctly performed, skipping"
+                        print("Cross section computation not correctly performed, skipping")
                         continue
             if os.path.isfile(json_output_folder+"/xsec_"+primary_dataset_name+".json"):
                 with open(json_output_folder+"/xsec_"+dataset.split('/')[1]+".json", 'r') as f:
@@ -41,11 +41,11 @@
                         and (not "\"MCM\": \"\"," in content) \
                         and (not "\"MCM\": \"None\"," in content): 
                         # and (not "CITo" in primary_dataset_name) \
-                        print "json OK, skipping"
+                        print("json OK, skipping")
                         continue
                     else:
-                        print "json corrupted, reproducing"
-                        print content
+                        print("json corrupted, reproducing")
+                        print(content)
             os.system("cp test_xsecdb_insert.json "+json_output_folder+"/xsec_"+primary_dataset_name+".json")
             os.system("sed -i -e 's/REPLACE_PRIMARY_DATASET_NAME/"+primary_dataset_name+"/g' "+json_output_folder+"/xsec_"+primary_dataset_name+".json")
             os.system("sed -i -e 's/REPLACE_FULL_DATASET_NAME/"+dataset.replace("/","\/")+"/g' "+json_output_folder+"/xsec_"+primary_dataset_name+".json")
@@ -102,7 +102,7 @@
                 os.system("sed -i -e 's/REPLACE_MCM_PREPID/"+str(mcm_prepid)+"/g' "+json_output_folder+"/xsec_"+primary_dataset_name+".json")
 
             with open(json_output_folder+"/xsec_"+dataset.split('/')[1]+".json", 'r+') as f:
-                print "produced json:\n", f.read()
+                print("produced json:\n", f.read())
             # sys.exit()
 
 

scripts/xsec_calc_auto/5_fill_xsec_database.py

@@ -11,28 +11,28 @@
     if not filename.endswith(".json"): 
         continue
     primary_dataset_name = filename.split("xsec_")[1].replace(".json","")
-    print primary_dataset_name
+    print(primary_dataset_name)
     # print filename
 
     if os.path.isfile(xsecdb_folder+"/update_logs/filldb_"+primary_dataset_name+".log"):
         with open(xsecdb_folder+"/update_logs/filldb_"+primary_dataset_name+".log", 'r') as f:
                     content = f.read()
                     if '{"status": "new"' in content and campaign in content: 
-                        print 'filldb report existing and valid, skipping dataset'
+                        print('filldb report existing and valid, skipping dataset')
                         continue
     if os.path.isfile(xsecdb_folder+"/update_logs/filldb_"+primary_dataset_name+"_"+campaign+".log"):
         with open(xsecdb_folder+"/update_logs/filldb_"+primary_dataset_name+"_"+campaign+".log", 'r') as f:
                     content = f.read()
                     if '{"status": "new"' in content and campaign in content: 
-                        print 'filldb report existing and valid, skipping dataset'
+                        print('filldb report existing and valid, skipping dataset')
                         continue
 
     with open(json_output_folder+filename, 'r') as f:
         content = f.read()
         if not 'REPLACE_' in content: 
-            print "json OK, uploading"
+            print("json OK, uploading")
         else:
-            print "json corrupted, skipping"
+            print("json corrupted, skipping")
             continue
 
     os.system(" cd "+xsecdb_folder+"/wrapper;\

server/api.py

@@ -1,7 +1,7 @@
 import json
 import re
 import copy
-import logger
+from . import logger
 
 from flask import Flask, request, jsonify, make_response, render_template
 from pymongo import MongoClient
@@ -10,13 +10,13 @@
 from time import gmtime, strftime
 from flask_cors import CORS
 
-from fields import fields as record_structure
-from mailing import send_mail, send_mail_approve
-from utils import compile_regex, get_ordered_field_list,\
+from .fields import fields as record_structure
+from .mailing import send_mail, send_mail_approve
+from .utils import compile_regex, get_ordered_field_list,\
     get_user_groups, is_user_in_group, get_field_order, remove_readonly_fields
-from validate import validate_model, validate_model_update
-from decorators import auth_user_group
-from config import CONFIG
+from .validate import validate_model, validate_model_update
+from .decorators import auth_user_group
+from .config import CONFIG
 
 app = Flask(__name__, static_folder="../client/dist",
             template_folder="../client/templates")
@@ -56,7 +56,7 @@ def get_by_id(record_id):
         structure = copy.deepcopy(record_structure)
 
         # Map record fields to information in record_structure (type, disabled, required, order)
-        for key, value in record.iteritems():
+        for key, value in record.items():
             if key in structure:
                 result_dic[key] = structure[key]  # type, disabled, title
                 # overwrite value with true value from db
@@ -216,7 +216,7 @@ def search():
 @app.route('/api/fields', methods=['GET'])
 def get_fields():
     """ get list of record_structure field names (for selecting visible columns) """
-    result = sorted(record_structure.keys(), key=get_field_order)
+    result = sorted(list(record_structure.keys()), key=get_field_order)
     return make_response(jsonify(result), 200)
 
 
@@ -229,7 +229,7 @@ def approve_records():
     logger.debug("APPROVE:" + str(record_ids) + " - USER " + user_login)
 
     curr_date = strftime("%Y-%m-%d %H:%M:%S", gmtime())
-    object_ids = map(lambda x: ObjectId(x), record_ids)
+    object_ids = [ObjectId(x) for x in record_ids]
     collection.update_many({'_id': {'$in': object_ids}}, {
                            '$set': {
                                'status': 'approved',

server/config.py

@@ -1,4 +1,4 @@
-import ConfigParser
+import configparser
 import os
 import json
 
@@ -9,7 +9,7 @@ def _update(self, **entries):
 app_dir = os.path.dirname(__file__)
 
 conf_dict = {}
-cfg_parser = ConfigParser.RawConfigParser()
+cfg_parser = configparser.RawConfigParser()
 a = cfg_parser.read(app_dir + "/config.cfg")
 
 for option in cfg_parser.options("strings"):

server/decorators.py

@@ -1,6 +1,6 @@
 from functools import wraps
 from flask import request, make_response, jsonify
-from utils import get_user_groups, is_user_in_group
+from .utils import get_user_groups, is_user_in_group
 
 # Decorator wrapped into function which accepts required e-group level
 def auth_user_group(group_level):

server/mailing.py

@@ -1,10 +1,10 @@
 import smtplib
-import logger
+from . import logger
 
 from email.MIMEMultipart import MIMEMultipart
 from email.MIMEText import MIMEText
 from email.Utils import COMMASPACE, formatdate
-from config import CONFIG
+from .config import CONFIG
 
 def send_mail(body, subject, recipients, **kwargs):
     sender = CONFIG.MAIL_SEND_FROM
@@ -22,7 +22,7 @@ def send_mail(body, subject, recipients, **kwargs):
         smtpObj.sendmail(sender, recipients, msg.as_string())
         smtpObj.close()
     except Exception as e:
-        print "Error: unable to send email", e.__class__, e.message
+        print("Error: unable to send email", e.__class__, e.message)
 
 
 def send_mail_approve(record_id):

server/utils.py

@@ -1,8 +1,8 @@
 import re
-import logger
+from . import logger
 from flask import request
-from config import CONFIG
-from fields import fields as record_structure
+from .config import CONFIG
+from .fields import fields as record_structure
 
 
 def get_user_groups():
@@ -38,7 +38,7 @@ def compile_regex(in_dic):
     '''
         compile value strings in mongo search query
     '''
-    for key, value in in_dic.iteritems():
+    for key, value in in_dic.items():
         if key != "$or" and key != "$and":
             in_dic[key] = re.compile(value, re.I)
         else:
@@ -63,7 +63,7 @@ def get_ordered_field_list(record_dict):
         transform record_structure dictionary into ordered list
     '''
     result = []
-    result_ = sorted(record_dict.iteritems(), key=lambda x: get_field_order(x[0]))
+    result_ = sorted(iter(record_dict.items()), key=lambda x: get_field_order(x[0]))
     for tupl in result_:
         dic = tupl[1]
         dic['name'] = tupl[0]
@@ -76,7 +76,7 @@ def remove_readonly_fields(record_request):
         remove read only fields from users request
     '''
 
-    read_only_fields = [key for key, value in record_structure.iteritems() if value.get('read_only', False)]
+    read_only_fields = [key for key, value in record_structure.items() if value.get('read_only', False)]
 
     for field_name in read_only_fields:
         record_request.pop(field_name, None)

server/validate.py

@@ -1,6 +1,6 @@
-import logger
+from . import logger
 import re
-from fields import fields as record_structure
+from .fields import fields as record_structure
 
 re_whitespace = re.compile(r'\s+')
 
@@ -11,7 +11,7 @@ def validate_model(record):
     error_obj = {}
 
     # validate against required fields from record structure
-    for key, field in record_structure.iteritems():
+    for key, field in record_structure.items():
         validate_field_(field, key=key, value=record.get(key), error_obj=error_obj)
 
     logger.debug(error_obj)
@@ -25,7 +25,7 @@ def validate_model_update(record_part):
     '''
     error_obj = {}
 
-    for key, value in record_part.iteritems():
+    for key, value in record_part.items():
         field = record_structure.get(key)
 
         validate_field_(field, key, value, error_obj)