DataProcessing/table.py at main · scg-wedo-iot/DataProcessing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
import pandas as pd
import os
import numpy as np
from .file import *

def loadCombineExport(pathType,pathLoad,pathSave,nameSave):

    if pathType == 'fullPath':
        listFil_dir = pathLoad
        nFiles = len(listFil_dir)
        for ifeat in range(nFiles):

            if ifeat == 0:
                dfFeatsAll = pd.read_csv(listFil_dir[ifeat])
            else:
                dfFeatsLoad = pd.read_csv(listFil_dir[ifeat])
                dfFeatsAll = pd.concat([dfFeatsAll, dfFeatsLoad], ignore_index=True)

        dfFeatsAll.to_csv( os.path.join(pathSave,nameSave) )

def filterTable(df, dictFilter):
    '''
    dictFilter = {
        "header_1": {
            "val": 99
            "inverse": True or False
        },
        "header_2": {
            "val": "A",
            "inverse": True or False
        }
    }

    :param df:
    :param dictFilter:
    :return:
    '''
    nrow = df.shape[0]
    isFirstItem = True
    for headerFil, dictVal in dictFilter.items():
        filterVal = dictVal['val']
        inverse = dictVal['inverse']

        if isinstance(filterVal, list):
            raise ValueError('filter valuse is list, can not process')

        if not inverse:
            indexFilterOut = df[headerFil].values == filterVal
        else:
            indexFilterOut = df[headerFil].values != filterVal

        if isFirstItem:
            indexFilterAll = indexFilterOut.copy()
            isFirstItem = False
        else:
            indexFilterAll = np.logical_and(indexFilterAll, indexFilterOut)

    dfFilter = df.loc[indexFilterAll, ]
    dfFilter.reset_index(inplace=True)

    return dfFilter, indexFilterAll


def read_csv_folder(pathFolder):
    listDir, listFolder, listName = findFile(pathFolder, '*.csv', 0)

    nFiles = len(listName)

    if nFiles == 1:
        df = pd.read_csv(listDir[0])

        return df

    elif nFiles > 1:
        listDF = []

        for ifile in range(len(listName)):
            df = pd.read_csv(listDir[ifile])

            listDF.append(df)

        return listDF

    else:
        df = []
        return df

def readCSVInfolder(pathFolder):
    listDir, listFolder, listName = findFile(pathFolder, '*.csv', 0)

    nFiles = len(listName)

    if nFiles == 1:
        df = pd.read_csv(listDir[0])

        return df

    elif nFiles > 1:
        listDF = []

        for ifile in range(len(listName)):
            df = pd.read_csv(listDir[ifile])

            listDF.append(df)

        return listDF

    else:
        df = []
        return df

def read_concat_write_csv_folder(folderLoad, folderSave, nameSave):
    listDir, listFolder, listName = findFile(folderLoad, '*.csv', 0)

    for ifile in range(len(listName)):
        df = pd.read_csv(listDir[ifile])

        if ifile == 0:
            dfAll = df
        else:
            dfAll = pd.concat((dfAll, df), axis=0)

    dfAll.to_csv(os.path.join(folderSave, nameSave))