Visualizing time of day and birth year

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

datadir = '/opt/data'
localdir = '/opt/program'
tripsdf = pd.read_csv(f'{datadir}/2013-07 - Citi Bike trip data.csv')
stationsdf = pd.read_csv(f'{localdir}/datas/stations/stations-2018-12-04-c.csv',
                        index_col=0)

tripsdf.iloc[0]

tripsdf['starttime'].map(lambda x: x[11:13]).iloc[:10]

tripsdf['hour'] = tripsdf['starttime'].map(lambda x: x[11:13])

# For all time looks like maybe two peaks

fig = plt.figure(figsize=(6, 6))
fig.patch.set_facecolor('xkcd:mint green')
ax = fig.add_subplot(111, )

ax.hist(tripsdf.hour.tolist(), bins=24)
plt.grid(True)

# Perhaps on weekdays different?
import fresh.utils as fu

fu.prepare_weekday_feature(tripsdf)

fig = plt.figure(figsize=(6, 6))
fig.patch.set_facecolor('xkcd:mint green')
ax = fig.add_subplot(111, )
ax.hist(tripsdf[tripsdf.weekday == True].hour.tolist(), bins=24)
ax.set_title('Weekday hour histogram')
plt.grid(True)

# Weekend wow big difference. 
fig = plt.figure(figsize=(6, 6))
fig.patch.set_facecolor('xkcd:mint green')
ax = fig.add_subplot(111, )
ax.hist(tripsdf[tripsdf.weekday == False].hour.tolist(), bins=24)
ax.set_title('Weekend hour histogram')
plt.grid(True)

# Ok based on the above, going to create a version 2...
# time_of_day_v2_peaky
# Not sure how to account for the different peaks on weekends
#     # 0: 6-10, 1: 11-15, 2: 16-20, 3: 21-00, 00-5

tripsdf[['usertype', 'birth year']].iloc[:5]

# Ok should basically fill these as np.nan for 'Customer'
dict(tripsdf[tripsdf.usertype=='Customer']['birth year'].value_counts())

# Luckily no nulls
tripsdf[(tripsdf.usertype=='Subscriber')& (tripsdf['birth year'].isnull())].shape

def make_xtick_labels(x, step=5):
    '''Given x, step the labels every <step>
    Aka, take every <step>th x label
    '''
    x_ticks = [i for i in  range(len(x)) if i % step == 0]
    x_labels = [x[i] for i in x_ticks]
    return x_ticks, x_labels

import numpy as np

tripsdf['birth year'] = tripsdf['birth year'].map(lambda x:int(x) if x != '\\N' else np.nan )

# Discard below 1913 as np.nan. TODO
tripsdf[(tripsdf.usertype=='Subscriber')& (tripsdf['birth year'] < 1913)].shape

X = [x for x in tripsdf[(tripsdf.usertype == 'Subscriber')]['birth year'].tolist()
    if x > 1913]

print(len(X))
fig = plt.figure(figsize=(12, 6))
fig.patch.set_facecolor('xkcd:mint green')
ax = fig.add_subplot(111, )

ax.hist(X[:10000], bins=70)

# x_ticks, x_labels = make_xtick_labels(x[:1000], step=20)
# ax.set_xticks(x_ticks)
# ax.set_xticklabels(x_labels, rotation=-45)

ax.set_title('Birth year binned (2013-07)')
plt.grid(True)
fig.show()

# Ok this seems normal ish. so might as well just split somewhat arbitrarily or evenly

def get_quantiles(unsorted):
    data = sorted(unsorted)
    minimum = data[0]
    Q1 = np.percentile(data, 25, interpolation = 'midpoint') 
    median = np.median(data)
    Q3 = np.percentile(data, 75, interpolation = 'midpoint') 
    maximum = data[-1]
    return [minimum, Q1, median, Q3, maximum]

def show_da_stats(bundle):
    H, bins = bundle['hist']
    quantiles = bundle['quantiles']
    fig = plt.figure(figsize=(12, 6))
    fig.patch.set_facecolor('xkcd:mint green')
    ax = fig.add_subplot(111, )
    ax.scatter(quantiles, [1, 1, 1, 1, 1])
    ax.axvline(quantiles[1], label='q:25%')
    ax.axvline(quantiles[2], label='q:50%')
    ax.axvline(quantiles[3], label='q:75%')

    ax.set_title(bundle['title'])
    ax.plot(bins, np.insert(H, 0, H[0]), drawstyle='steps', color='green')
    plt.grid(True)
    fig.legend()
    fig.show()
    
#
hist = np.histogram(X, bins=100, range=None)

quantiles = get_quantiles(X)
bundle = {'hist': hist, 'quantiles': quantiles,
               'title': 'Birth year binned (2013-07)'}
print('quantiles', quantiles)
show_da_stats(bundle)

# And the age quantiles to make this a year independent feature.. 
2013 - np.array([1920.0, 1969.0, 1978.0, 1984.0, 1997.0])

import datetime; import pytz
x = dict(tripsdf.iloc[2])
x

(x['start_dt'] - 
     datetime.datetime(int(x['birth year']), 1, 1, tzinfo=pytz.timezone('US/Eastern'))
).days/365. , (x['start_dt'] - np.nan)

x['start_dt'] - pd.NaT

# Reload once again since I modified a col
tripsdf = pd.read_csv(f'{datadir}/2013-07 - Citi Bike trip data.csv')

reload(fu)
minidf = tripsdf.iloc[:1000].copy()
fu.prepare_weekday_feature(minidf)
fu.age_feature(minidf)

list(reversed([93., 44., 35., 29., 16.]))

minidf['birth_bin'] = pd.cut(minidf['age'], bins=[16.0, 29.0, 35.0, 44.0, 93.0],
                            labels=[0, 1, 2, 3])

minidf[['age', 'birth', 'birth_bin']].iloc[:10]

import fresh.predict_utils as fpu

bundle = fpu.load_bundle_in_docker()

print('original proc bundle notebook', bundle['proc_bundle']['bundle']['notebook'])
bundle['proc_bundle']['bundle']['proc_bundle'].keys()
print('this is the bundle glue notebook, cool', bundle['notebook'])
print('model notebook', bundle['model_bundle']['bundle']['notebook'])
print('train', bundle['model_bundle']['bundle']['train'])
print('test', bundle['model_bundle']['bundle']['validation_metrics']['test'])

import fresh.preproc.v3 as pv3

!pwd