# imports that I'll use over the course of the project
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import cross_val_score, cross_val_predict, train_test_split, KFold
from sklearn.linear_model import LinearRegression, Lasso
from sklearn.metrics import r2_score

%matplotlib inline

# read in data
houses = pd.read_csv('./data/train.csv')
holdout_set = pd.read_csv('./data/test.csv')

# Check out the data types
houses.dtypes

# Basic description to check any weirdness
houses.describe().T

# Understand the size of the data
houses.shape

# Null value check
df_nulls = pd.DataFrame(data=houses.isnull().sum(), columns=['Nulls'])
df_nulls.sort_values('Nulls', ascending=False)
# There seems to be quite a few null values, should look at cleaning these up.

# If and when I wanna drop all na's from original DF:
# houses.dropna(axis=0, how='any')
# houses.select_dtypes(object)

# Setup dummy variables for later
houses_object_columns = pd.get_dummies(houses,columns=houses.select_dtypes(object).columns)

houses_object_columns.head()

df_nulls_obj = pd.DataFrame(data=houses_object_columns.isnull().sum(), columns=['Nulls'])
df_nulls_obj.sort_values('Nulls', ascending=False)

# Clean up NAN
houses_object_columns.fillna(value=0, inplace=True)

# Most of the code after the features section was run multiple times to attempt to find the best model

# Original run of feature engineering
# Just randomly picking a few potential predictors
features = ['Lot Area', 'Overall Qual','Fireplaces','TotRms AbvGrd']
X = houses[features]
y = houses['SalePrice']

# I'm going to pull out only numeric columns, to see if there's some variables with high correlation that I can throw in to improve my model.
# Make it into it's own DF to run some EDA on it
houses_numonly = houses.select_dtypes(np.number)

# houses_numonly_dropna = houses_numonly.dropna(axis=0, how='any')
# houses_numonly_dropna.isnull().sum()

# Original run
# houses_numonly.columns
houses_numonly.shape

# For original run of determining correlations
features = houses_numonly.columns
X = houses[features]
y = houses_numonly['SalePrice']

# X.corr() is far too painful to look at here
# Run the heatmap, see if anything sticks out
plt.subplots(figsize=(40,30))
sns.heatmap(X.corr(), annot=True)

# Just eyeballing it, some important correlations I see (descending):
# I stopped at 'Garage Yr Blt', which had a correlation of .53
# Upon further review, 'Garage Yr Blt' had far too many missing data points, so was left off
features = ['Overall Qual',
'Gr Liv Area',
'Garage Cars',
'Garage Area',
'Total Bsmt SF',
'1st Flr SF',
'Year Built',
'Year Remod/Add',
'Full Bath']
# 'Garage Yr Blt']

# Third time around: going to add a few variables - might cause overfit, but let's see
# Was more lax on what variables I added (based on correlation > .3)
features = ['Overall Qual',
'Gr Liv Area',
'Garage Cars',
'Garage Area',
'Total Bsmt SF',
'1st Flr SF',
'Year Built',
'Year Remod/Add',
'Full Bath',
'TotRms AbvGrd',
'Mas Vnr Area',
'Fireplaces',
'BsmtFin SF 1',
'Wood Deck SF']

# Set your in/dependent variables
# Original usage
# X = houses[features]
# y = houses['SalePrice']

# Old run:
# X = houses_object_columns[features]
# y = houses_object_columns['SalePrice']

# Null check
X.isnull().sum()

# Fill in the nas:
# Despite the warning, this still works:
X['Mas Vnr Area'].fillna(value=0, inplace=True)
X['Garage Cars'].fillna(value=0, inplace=True)
X['Garage Area'].fillna(value=0, inplace=True)
X['Total Bsmt SF'].fillna(value=0, inplace=True)
X['BsmtFin SF 1'].fillna(value=0, inplace=True)

# Check:
X.isnull().sum()

# X.shape[0]

# 2nd feature run note: Going to rebuild my features, minus Grg yr blt as there seems to be more than a few nulls
# Some of the next few cells were only used for original run through of cleaning up features
# Take a look at null values in my features:
# X[X['Garage Cars'].isnull() == True]
# Looks like the singular null in 'Grg Cars' is one in the same w/ 'Grg Area'

# # Dropping the rows with the null values
# X = X.drop(X.index[1712])
# # Sets have to match in row length:
# y = y.drop(y.index[1712])

# Check to make sure it worked
# X[X['Garage Cars'].isnull() == True]

# X[X['Total Bsmt SF'].isnull() == True]

# Dropping the rows with the null values
# X = X.drop(X.index[1327])
# Sets have to match in row length:
# y = y.drop(y.index[1327])

# Check to make sure it worked
# X[X['Total Bsmt SF'].isnull() == True]

# Messing around trying to figure out how to locate the null rows
# X.columns[X.isna().any()].tolist()
# X.loc[:,X.isna().any()]

X = houses_object_columns.drop('SalePrice', axis=1)
y = houses_object_columns['SalePrice']

# train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y)

X_train.fillna(value=0, inplace=True)
# X_test.fillna(value=0, inplace=True)
y_train.fillna(value=0, inplace=True)
# y_test.fillna(value=0, inplace=True)

L = Lasso(alpha = 2.5, max_iter=10000, random_state = 42)

L.fit(X_train, y_train)

# Original attempts:
# LinearRegression
# linreg = LinearRegression()
# Original fit
# linreg.fit(X, y)
# linreg.coef_

# Using KFold to help randomize the folds, and also see if different splits get me significantly different scores
kf = KFold(n_splits=5, random_state=42, shuffle=True)

scores = cross_val_score(L, X_train, y_train, cv=kf)
print(scores)
print(scores.mean())

# Overfit
L.score(X_test, y_test)

predictions = cross_val_predict(L, X, y, cv=kf)
plt.scatter(y, predictions)
accuracy = r2_score(y, predictions)

# Looks like my model is overfit
# I ran this a few times, and didn't get nearly as bad of number. Looks like there was one fold that my model was just completely ill-fitting on.
scores = cross_val_score(L, X_test, y_test, cv=kf)
print(scores)
print(scores.mean())

# Clean up and make changes along the same lines to holdout_set
holdout_set.fillna(value=0, inplace=True)

holdout_set.head(10)

X_holdout = holdout_set[features]

X_holdout.head(10)

X_holdout.isnull().sum()

# Replace one NA in Mas VNR Area
X_holdout.fillna(value=0, inplace=True)

# Original linreg
y_preds = linreg.predict(X_holdout)

# Lasso run:
y_preds = L.predict(X_holdout)

y_preds

my_ids = holdout_set['Id']

df = pd.DataFrame()

df['Id'] = my_ids

df.set_index('Id', inplace=True)

df['SalePrice'] = y_preds

%pwd

df.to_csv('./data/my_preds.csv')

# Was messing around trying to find a loop that could get me all numeric columns:
# .select_dtypes solved this issue
list_house_intFloat = []
for column in dfhouses.columns:
    if dfhouses[column].dtypes == int:
#         print('is int')
        list_house_intFloat.append(dfhouses[column])
#     elif dfhouses[column].dtypes == float:
#         dfhouse_int.append(dfhouses.loc[dfhouses[column]]

# Counts for categorization of Street or Neighborhood
houses['Street'].value_counts()

# Drop na's - was using before I did it on the original set
# X.dropna(axis=0, how='any', inplace=True)

# houses_numonly_dropna = houses_numonly.dropna(axis=0, how='any')
# houses_numonly_dropna.isnull().sum()