From 998f4e9035d73cf2876f793b993f729ef718627d Mon Sep 17 00:00:00 2001 From: Andy Wu Date: Fri, 4 Oct 2019 22:44:53 -0700 Subject: [PATCH 1/4] code refactoring, ebs volume extension for testnet --- devops/extend_ebs_volume_testnet/main.py | 111 +++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 devops/extend_ebs_volume_testnet/main.py diff --git a/devops/extend_ebs_volume_testnet/main.py b/devops/extend_ebs_volume_testnet/main.py new file mode 100644 index 00000000..7d8a086f --- /dev/null +++ b/devops/extend_ebs_volume_testnet/main.py @@ -0,0 +1,111 @@ +import os +import sys +import boto3 +import botocore +from botocore.exceptions import ClientError + +EBS_size = 40 +BUCKET_NAME = 'harmony-benchmark' +region_array = ['us-east-1', 'us-west-2'] + +path_file_s3 = 'logs/2019/09/11/005018/' +remote_dist_file = path_file_s3 + 'distribution_config.txt' + +home_dir = os.getcwd() +temp_dir = home_dir + '/temp' +try: + os.mkdir(temp_dir) +except OSError: + print("Director exists, or creation of this directory %s failed" % temp_dir ) +else: + print("Successfully created this directory %s " % temp_dir) + +local_dist_file = temp_dir + '/distribution_config.txt' + +dict_ip_per_region = {} +dict_ebs_id_region = {} + +session = boto3.Session(profile_name='default') + +def download_file_s3(bucketname, remote_file, local_file): + s3_client = boto3.resource('s3') + s3_client.Bucket(bucketname).download_file(remote_file, local_file) + +def create_ip_for_each_region(fpath): + if not os.path.isfile(fpath): + print("File path {} does not exist. Exiting...".format(fpath)) + sys.exit() + + fp = open(fpath) + line = fp.readline() + + while line: + if line.rstrip(): + ip = line.split()[0] + region_num = line.split()[4][0] + if region_num not in dict_ip_per_region.keys(): + dict_ip_per_region[region_num] = [ip] + else: + dict_ip_per_region[region_num].append(ip) + line = fp.readline() + fp.close() + +def create_ebs_volume_id_each_region(dict_ip): + for region, ip_array in dict_ip.items(): + if region == '1': + ec2 = session.client('ec2', region_name = 'us-east-1') + for ip in ip_array: + try: + resp = ec2.describe_instances(Filters=[{'Name': 'ip-address', 'Values': [ip]}]) + if len(resp["Reservations"]) != 0: + vol_id = resp["Reservations"][0]["Instances"][0]["BlockDeviceMappings"][0]["Ebs"]["VolumeId"] + if region not in dict_ebs_id_region.keys(): + dict_ebs_id_region[region] = [vol_id] + else: + dict_ebs_id_region[region].append(vol_id) + except ClientError as e: + print("Unexpected error: %s" % e) + elif region == '4': + ec2 = session.client('ec2', region_name = 'us-west-2') + for ip in ip_array: + try: + resp = ec2.describe_instances(Filters=[{'Name': 'ip-address', 'Values': [ip]}]) + if len(resp["Reservations"]) != 0: + vol_id = resp["Reservations"][0]["Instances"][0]["BlockDeviceMappings"][0]["Ebs"]["VolumeId"] + if region not in dict_ebs_id_region.keys(): + dict_ebs_id_region[region] = [vol_id] + else: + dict_ebs_id_region[region].append(vol_id) + except ClientError as e: + print("Unexpected error: %s" % e) + +def increase_ebs_volume(dict_ebs): + for region, ebs_id_array in dict_ebs.items(): + if region == '1': + ec2 = session.client('ec2', region_name = 'us-east-1' ) + for vol_id in ebs_id_array: + try: + resp = ec2.modify_volume(VolumeId = vol_id, Size = EBS_size) + except ClientError as e: + print("Unexpected error: %s" % e) + if region == '4': + ec2 = session.client('ec2', region_name = 'us-west-2' ) + for vol_id in ebs_id_array: + try: + resp = ec2.modify_volume(VolumeId = vol_id, Size = EBS_size) + except ClientError as e: + print("Unexpected error: %s" % e) + +def main(): + + download_file_s3(BUCKET_NAME, remote_dist_file, local_dist_file) + + create_ip_for_each_region(local_dist_file) + + create_ebs_volume_id_each_region(dict_ip_per_region) + + increase_ebs_volume(dict_ebs_id_region) + + +if __name__ == "__main__": + main() From 6f66dfb93e23bfb44a6448e565743f2702859b3c Mon Sep 17 00:00:00 2001 From: Andy Wu Date: Mon, 4 Nov 2019 11:50:00 -0800 Subject: [PATCH 2/4] start code refactoring for ebs volume extension --- devops/extend_ebs_volume_testnet/main.py | 98 +++--------------------- 1 file changed, 10 insertions(+), 88 deletions(-) diff --git a/devops/extend_ebs_volume_testnet/main.py b/devops/extend_ebs_volume_testnet/main.py index 7d8a086f..548245b2 100644 --- a/devops/extend_ebs_volume_testnet/main.py +++ b/devops/extend_ebs_volume_testnet/main.py @@ -4,107 +4,29 @@ import botocore from botocore.exceptions import ClientError -EBS_size = 40 -BUCKET_NAME = 'harmony-benchmark' -region_array = ['us-east-1', 'us-west-2'] +NETWORK = 'mainnet' -path_file_s3 = 'logs/2019/09/11/005018/' -remote_dist_file = path_file_s3 + 'distribution_config.txt' +# array to store list of ip files to download from github +ip_file_array = [] -home_dir = os.getcwd() -temp_dir = home_dir + '/temp' -try: - os.mkdir(temp_dir) -except OSError: - print("Director exists, or creation of this directory %s failed" % temp_dir ) -else: - print("Successfully created this directory %s " % temp_dir) -local_dist_file = temp_dir + '/distribution_config.txt' -dict_ip_per_region = {} -dict_ebs_id_region = {} +def create_file_array(network): + if network == 'mainnet': + ip_file_array = [] + elif network == 'testnet': + ip_file_array = [] -session = boto3.Session(profile_name='default') -def download_file_s3(bucketname, remote_file, local_file): - s3_client = boto3.resource('s3') - s3_client.Bucket(bucketname).download_file(remote_file, local_file) -def create_ip_for_each_region(fpath): - if not os.path.isfile(fpath): - print("File path {} does not exist. Exiting...".format(fpath)) - sys.exit() - fp = open(fpath) - line = fp.readline() - - while line: - if line.rstrip(): - ip = line.split()[0] - region_num = line.split()[4][0] - if region_num not in dict_ip_per_region.keys(): - dict_ip_per_region[region_num] = [ip] - else: - dict_ip_per_region[region_num].append(ip) - line = fp.readline() - fp.close() - -def create_ebs_volume_id_each_region(dict_ip): - for region, ip_array in dict_ip.items(): - if region == '1': - ec2 = session.client('ec2', region_name = 'us-east-1') - for ip in ip_array: - try: - resp = ec2.describe_instances(Filters=[{'Name': 'ip-address', 'Values': [ip]}]) - if len(resp["Reservations"]) != 0: - vol_id = resp["Reservations"][0]["Instances"][0]["BlockDeviceMappings"][0]["Ebs"]["VolumeId"] - if region not in dict_ebs_id_region.keys(): - dict_ebs_id_region[region] = [vol_id] - else: - dict_ebs_id_region[region].append(vol_id) - except ClientError as e: - print("Unexpected error: %s" % e) - elif region == '4': - ec2 = session.client('ec2', region_name = 'us-west-2') - for ip in ip_array: - try: - resp = ec2.describe_instances(Filters=[{'Name': 'ip-address', 'Values': [ip]}]) - if len(resp["Reservations"]) != 0: - vol_id = resp["Reservations"][0]["Instances"][0]["BlockDeviceMappings"][0]["Ebs"]["VolumeId"] - if region not in dict_ebs_id_region.keys(): - dict_ebs_id_region[region] = [vol_id] - else: - dict_ebs_id_region[region].append(vol_id) - except ClientError as e: - print("Unexpected error: %s" % e) - -def increase_ebs_volume(dict_ebs): - for region, ebs_id_array in dict_ebs.items(): - if region == '1': - ec2 = session.client('ec2', region_name = 'us-east-1' ) - for vol_id in ebs_id_array: - try: - resp = ec2.modify_volume(VolumeId = vol_id, Size = EBS_size) - except ClientError as e: - print("Unexpected error: %s" % e) - if region == '4': - ec2 = session.client('ec2', region_name = 'us-west-2' ) - for vol_id in ebs_id_array: - try: - resp = ec2.modify_volume(VolumeId = vol_id, Size = EBS_size) - except ClientError as e: - print("Unexpected error: %s" % e) def main(): - download_file_s3(BUCKET_NAME, remote_dist_file, local_dist_file) - - create_ip_for_each_region(local_dist_file) - create_ebs_volume_id_each_region(dict_ip_per_region) + create_file_array(NETWORK) - increase_ebs_volume(dict_ebs_id_region) + download_ipfiles_github(NETWORK) if __name__ == "__main__": From 58e1698d8359dd06330e029512a825cf7dbc72ae Mon Sep 17 00:00:00 2001 From: Andy Wu Date: Mon, 4 Nov 2019 13:17:04 -0800 Subject: [PATCH 3/4] add git_token def --- devops/extend_ebs_volume_testnet/main.py | 50 +++++++++++++++++++----- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/devops/extend_ebs_volume_testnet/main.py b/devops/extend_ebs_volume_testnet/main.py index 548245b2..7df43f08 100644 --- a/devops/extend_ebs_volume_testnet/main.py +++ b/devops/extend_ebs_volume_testnet/main.py @@ -3,30 +3,62 @@ import boto3 import botocore from botocore.exceptions import ClientError +import subprocess +from dotenv import load_dotenv +from timeit import default_timer as timer +from datetime import timedelta +# which network would you like update storage? mainnet or testnet? NETWORK = 'mainnet' + + + # array to store list of ip files to download from github ip_file_array = [] +load_dotenv() +GIT_TOKEN = os.getenv('GIT_TOKEN') - -def create_file_array(network): +def create_ip_file_array(network): if network == 'mainnet': - ip_file_array = [] + path_mainnet_github = 'https://raw.githubusercontent.com/harmony-one/nodedb/master/mainnet/' + path_s0 = path_mainnet_github + 'shard0.txt' + path_s1 = path_mainnet_github + 'shard1.txt' + path_s2 = path_mainnet_github + 'shard2.txt' + path_s3 = path_mainnet_github + 'shard3.txt' + return [path_s0, path_s1, path_s2, path_s3] elif network == 'testnet': - ip_file_array = [] - - + path_testnet_github = 'https://raw.githubusercontent.com/harmony-one/nodedb/master/testnet/' + path_s0 = path_testnet_github + 'shard0.txt' + path_s1 = path_testnet_github + 'shard1.txt' + path_s2 = path_testnet_github + 'shard2.txt' + return [path_s0, path_s1, path_s2] +def shcmd(cmd, ignore_error=False): + # print('Doing:', cmd) + ret = subprocess.call(cmd, shell=True) + # print('Returned', ret, cmd) + if ignore_error == False and ret != 0: + raise RuntimeError("Failed to execute {}. Return code:{}".format( + cmd, ret)) + return ret +def download_ipfiles_github(ip_array): + for file in ip_array: + cmd = "curl -H 'Authorization: token {token}' "\ + "-H 'Accept: application/vnd.github.v3.raw' -o /tmp/shard0.txt "\ + "{path}".format(token=GIT_TOKEN, path=file) + shcmd(cmd) def main(): + # create an array to store the path to download ip files + ip_file_array = create_ip_file_array(NETWORK) + print(ip_file_array) - create_file_array(NETWORK) - - download_ipfiles_github(NETWORK) + # download ip files from github + download_ipfiles_github(ip_file_array) if __name__ == "__main__": From 07f0cfb0a14906032b90a824c3345e6f14287a86 Mon Sep 17 00:00:00 2001 From: Andy Wu Date: Fri, 8 Nov 2019 10:10:30 -0800 Subject: [PATCH 4/4] with major code refactoring, and support two aws profiles --- devops/extend_ebs_volume_testnet/.gitignore | 2 + devops/extend_ebs_volume_testnet/main.py | 149 ++++++++++++++++++-- 2 files changed, 137 insertions(+), 14 deletions(-) create mode 100644 devops/extend_ebs_volume_testnet/.gitignore diff --git a/devops/extend_ebs_volume_testnet/.gitignore b/devops/extend_ebs_volume_testnet/.gitignore new file mode 100644 index 00000000..08298b52 --- /dev/null +++ b/devops/extend_ebs_volume_testnet/.gitignore @@ -0,0 +1,2 @@ +.env +*.txt diff --git a/devops/extend_ebs_volume_testnet/main.py b/devops/extend_ebs_volume_testnet/main.py index 7df43f08..1276ef8c 100644 --- a/devops/extend_ebs_volume_testnet/main.py +++ b/devops/extend_ebs_volume_testnet/main.py @@ -1,6 +1,8 @@ import os +import pprint import sys import boto3 +import time import botocore from botocore.exceptions import ClientError import subprocess @@ -11,55 +13,174 @@ # which network would you like update storage? mainnet or testnet? NETWORK = 'mainnet' - - +EBS_SIZE = 200 # array to store list of ip files to download from github ip_file_array = [] +# a dict to store ips per region +dict_region_ip_array = {} + +# a dict to store vol-id per region +dict_region_volid_array = {} + load_dotenv() GIT_TOKEN = os.getenv('GIT_TOKEN') +profile_array = ['default', 'mainnet-aw'] + def create_ip_file_array(network): if network == 'mainnet': path_mainnet_github = 'https://raw.githubusercontent.com/harmony-one/nodedb/master/mainnet/' - path_s0 = path_mainnet_github + 'shard0.txt' + #path_s0 = path_mainnet_github + 'shard0.txt' path_s1 = path_mainnet_github + 'shard1.txt' path_s2 = path_mainnet_github + 'shard2.txt' path_s3 = path_mainnet_github + 'shard3.txt' - return [path_s0, path_s1, path_s2, path_s3] + return [path_s1, path_s2, path_s3] elif network == 'testnet': path_testnet_github = 'https://raw.githubusercontent.com/harmony-one/nodedb/master/testnet/' - path_s0 = path_testnet_github + 'shard0.txt' + #path_s0 = path_testnet_github + 'shard0.txt' path_s1 = path_testnet_github + 'shard1.txt' path_s2 = path_testnet_github + 'shard2.txt' - return [path_s0, path_s1, path_s2] + return [path_s1, path_s2] def shcmd(cmd, ignore_error=False): - # print('Doing:', cmd) ret = subprocess.call(cmd, shell=True) - # print('Returned', ret, cmd) if ignore_error == False and ret != 0: raise RuntimeError("Failed to execute {}. Return code:{}".format( cmd, ret)) return ret +def shcmd2(cmd, ignore_error=False): + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + output = proc.stdout.read() + output_string = output.decode("utf-8") + return output_string + def download_ipfiles_github(ip_array): for file in ip_array: cmd = "curl -H 'Authorization: token {token}' "\ - "-H 'Accept: application/vnd.github.v3.raw' -o /tmp/shard0.txt "\ + "-H 'Accept: application/vnd.github.v3.raw' -O "\ "{path}".format(token=GIT_TOKEN, path=file) shcmd(cmd) +def create_array_to_keep_all_ip(): + all_ip = [] + current_dir = os.path.dirname(os.path.realpath(__file__)) + # TO-DO: mainnet only + local_shard_file_array = [current_dir+"/shard1.txt", current_dir+"/shard2.txt", current_dir+"/shard3.txt"] + print(local_shard_file_array) + for item in local_shard_file_array: + fp = open(item) + line = fp.readline() + while line: + line = line.rstrip() + all_ip.append(line) + line = fp.readline() + fp.close() + return all_ip + +def get_region_from_public_ip(ip): + region = '' + try: + cmd = "host -W 20 {ip}".format(ip=ip) + resp = shcmd2(cmd) + info = resp.split('.') + print(info) + if info[-4] == 'compute': + region = info[-5] + elif info[-4] == 'compute-1': + region = 'us-east-1' + else: + raise ValueError("cannot deduce region from name {}".format(info)) + except Exception as e: + print("Unexpected error: %s" % e) + + return region + +def create_ip_dict_per_region(ip_array): + dict_region_ip = {} + for ip in ip_array: + time.sleep(2) + region = get_region_from_public_ip(ip) + if region not in dict_region_ip.keys(): + dict_region_ip[region] = [ip] + else: + dict_region_ip[region].append(ip) + + return dict_region_ip + +def update_dict_region_ips(reg, ips, dict_region_volid_array): + + # our nodes are located in two different AWS accounts + # legacy nodes are located in aws-6565 account; while tf nodes are located in aws-7746 account + # + for profile in profile_array: + print("The PROFILE is: ", profile) + session = boto3.Session(profile_name=profile) + + ec2 = session.client('ec2', region_name = reg) + for ip in ips: + try: + resp = ec2.describe_instances(Filters=[{'Name': 'ip-address', 'Values': [ip]}]) + print(resp) + if len(resp["Reservations"]) != 0: + vol_id = resp["Reservations"][0]["Instances"][0]["BlockDeviceMappings"][0]["Ebs"]["VolumeId"] + if reg not in dict_region_volid_array.keys(): + dict_region_volid_array[reg] = [vol_id] + else: + dict_region_volid_array[reg].append(vol_id) + except ClientError as e: + print("Unexpected error: %s" % e) + + +def create_volid_dict_per_region(dict_region_ips): + for region, ip_array in dict_region_ips.items(): + print("region - fetch vol id: ", region) + update_dict_region_ips(region, ip_array, dict_region_volid_array) + + +def extend_ebs_per_region(reg, vols): + for profile in profile_array: + session = boto3.Session(profile_name=profile) + + ec2 = session.client('ec2', region_name = reg) + for vol_id in vols: + try: + resp = ec2.modify_volume(VolumeId = vol_id, Size = EBS_SIZE) + print(resp) + except ClientError as e: + print("Unexpected error: %s" % e) + +def extend_ebs_volumes(dict_region_volid): + for region, volid_array in dict_region_volid.items(): + print("region - extend ebs vol: ", region) + extend_ebs_per_region(region, volid_array) + def main(): - # create an array to store the path to download ip files - ip_file_array = create_ip_file_array(NETWORK) - print(ip_file_array) + # # create an array to store the path to download ip files + # ip_file_array = create_ip_file_array(NETWORK) + # + # # download ip files from github + # download_ipfiles_github(ip_file_array) + + # load all ip files into a single array + array_total_ip = create_array_to_keep_all_ip() + + # create a dict to store ip per region + dict_region_ip_array = create_ip_dict_per_region(array_total_ip) + print("-- dict_region_ip_array") + pprint.pprint(dict_region_ip_array) - # download ip files from github - download_ipfiles_github(ip_file_array) + # create a dict to store vol_id per region + create_volid_dict_per_region(dict_region_ip_array) + print("-- dict_region_volid_array") + print(len(dict_region_volid_array)) + pprint.pprint(dict_region_volid_array) + # aws api call to increase storage volume + extend_ebs_volumes(dict_region_volid_array) if __name__ == "__main__": main()