import csv
from random import randint, choice, random
import datetime
import os

from scipy.stats import norm

def generate_spoof_file(title, date, values=180):
    """
    Generate a CSV file holding mock data for us to play with
    """
    options = ('FULL','PARTIAL','FAIL')
    parameters = [str(randint(-5,5)) for _ in range(5)]
    parameters = ",".join(parameters)

    with open(title, 'w', newline='') as csvfile:
        writer = csv.writer(csvfile, delimiter=";", quotechar="\'")
        writer.writerow(['Expensive Apparatus'])
        writer.writerow(['Param :: ({})'.format(parameters)])
        writer.writerow(['Date :: {}'.format(date)])
        writer.writerow([])
        writer.writerow(['id','time','alter','value','profile',''])
        
        
        val = norm(loc=5, scale=0.5).rvs(values)
        time = datetime.datetime(year=2019, month=5, day=21, hour=3, minute=11)
        for j in range(values):
            time += datetime.timedelta(minutes=1) 
            writer.writerow([j,time.time(),randint(0,100), "{:.5f}".format(val[j]), choice(options)])

def generate_files(loc='input', num_of_files=10):
    """
    Generates spoof files in a subdirectory
    """
    try:
        os.mkdir(loc)
    except FileExistsError:
        print("Directory already exists")

    while len(os.listdir(loc)) < num_of_files:
        word = choice(('typical', 'important', 'good', 'weird'))
        date = "{day}{month}2018".format(day=randint(1,28), month=randint(1,12))
        filename = "{}_{}_run{}.csv".format(word, date, choice('AB'))
        print("Creating", filename)
        location = os.path.join(loc, filename)
        generate_spoof_file(location, date=date)
        
generate_files()

with open('input/good_2432018_runA.csv') as f:
    for j, row in enumerate(f):
        if j > 10:
            break
        print(j, row, end='')

import os
root_directory = 'input'
files_to_parse = {}  # filename : path

for root, dirs, files in os.walk(root_directory):
    for fn in files:
        if fn.endswith('csv'):
            files_to_parse[fn] = os.path.join(root, fn)
            
files_to_parse

line_seperator = ';'

for filename in files_to_parse:
    print(f'Parsing : {filename}')
    with open(files_to_parse[filename]) as f:
        for j, line in enumerate(f):
            line = line.rstrip('\r\n')  # get rid of tabs and newlines at end of rows
            if j > 3:
                line = line.split(line_seperator)
                line = [itm for itm in line if itm]  # get rid of empty spaces
                
            print(j, line)
            if j > 10:
                break
    break

from dateutil import parser

def parse_datetime(time):
    return parser.parse(time).time()



def parse_file(location, line_seperator = ';', **parameters):
    '''
    Parse file using information discussed
    
    '''
    _rows = {'id' : int, 'time' :  parse_datetime, 'alter' : int, 'value' : float, 'profile' : str}
    rows = parameters.get('rows', _rows)
    
    with open(location) as f:
        for j, line in enumerate(f):
            line = line.rstrip('\r\n')  # get rid of tabs and newlines at end of rows
            if j > 4:
                line = line.split(line_seperator)
                line = [itm for itm in line if itm]  # get rid of empty spaces
                line = {lbl : rows[lbl](value) for value, lbl in zip(line, rows.keys())}
                yield line
    
loaded_data = {filename : list(parse_file(files_to_parse[filename])) for filename in files_to_parse}

fn = 'good_2132018_runA.csv'
print(f'Data from {fn}')
for j, row in enumerate(loaded_data[fn]):
    print(j, row)
    
    if j >= 10:
        break

from statistics import mean, stdev

for filename in loaded_data:
    for profile in ('FAIL', 'FULL', 'PARTIAL'):
        data = [row['value'] for row in loaded_data[filename] if row['profile'] == profile]
        mean_value = mean(data)
        std_value = stdev(data)
        print(f"{filename}, {profile} has a mean of {mean_value:.2f} +/- {std_value:.2f}")
    break  # only prints first filename!

import pandas as pd

filename = 'good_2132018_runA.csv'
location = files_to_parse[filename]

df = pd.read_csv(location, index_col=0, skiprows=5, sep=';', parse_dates=[1],
                 names=('id', 'time', 'alter', 'value', 'profile'),
                )

df.time = df.time.apply(lambda row : row.time())  # get rid of date bit

df.head()

filename = 'good_2132018_runA.csv'

for profile, grp in df.groupby('profile'):
    mean_value = grp.value.mean()
    std_value = grp.value.std()
    print(f"{filename}, {profile} has a mean of {mean_value:.2f} +/- {std_value:.2f}")

from scipy.stats import norm
from matplotlib.pyplot import subplots, show
from numpy import linspace

x = linspace(3, 7)
for profile, grp in df.groupby('profile'):
    
    loc, scale = norm.fit(grp.value)
    print(f'{loc:.2f} +/- {scale:.2f}')

    fig, ax = subplots()
    ax.set_title(profile)
    
    ax.plot(x, norm.pdf(x, loc=loc, scale=scale))
    data = grp.value.hist(density=1,ax=ax)

    show()