import pandas as pd
import numpy as np
pd.set_option("display.max_columns", None)

train = pd.read_csv("E:\\kaggle\\House Prices Advanced Regression Techniques\\Data\\train.csv")
test = pd.read_csv("E:\\kaggle\\House Prices Advanced Regression Techniques\\Data\\test.csv")

train.head()

import matplotlib.pyplot as plt
%matplotlib inline
## Neighborhood ：表示的是房屋建筑所在的地理位置。
nei_pivot = train.pivot_table(index="Neighborhood", values="SalePrice", aggfunc=np.median)
## 排下序
nei_pivot_sorted = nei_pivot.sort_values(by="SalePrice")
nei_pivot_sorted.plot(kind="bar", color="blue")
plt.xlabel("Neighborhood")
plt.ylabel("sale price")
plt.show()

from IPython.display import Image
# 草甸村
Image("meadow_village.jpg")

# 北岭高地
Image("Northridge_Heights.jpg")

## OverallCond：评价房子的整体情况
overallCond_pivot = train.pivot_table(index="OverallCond", values="SalePrice", aggfunc=np.median)
overallCond_pivot.plot(kind="bar", color="blue")
plt.xlabel("OverallCond")
plt.ylabel("SalePrice")
plt.show()

## OverallQual: 房子建造的成本
overallQual_pivot = train.pivot_table(index="OverallQual", values="SalePrice", aggfunc=np.median)
overallQual_pivot.plot(kind="bar", color="blue")
plt.xlabel("OverallQual")
plt.ylabel("SalePrice")
plt.show()

## 售价
train.SalePrice.describe()

price_describe = train.SalePrice.describe().drop("count")
price_describe.plot(kind="bar", color="blue")

for i, item in enumerate(price_describe):
    plt.text(i, item, "{}".format(int(item)), ha='center', va='bottom')
plt.show()

train.SalePrice.plot(kind="hist")
plt.show()

print(train.SalePrice.skew())

normal_dis_price = np.log(train.SalePrice)
normal_dis_price.plot(kind="hist")
plt.show()

## 利用 .select_dtypes() 方法可以从DataFrame中将你选中的数据筛选出来
numeric_features = train.select_dtypes(include=[np.number])

corr = numeric_features.corr()

print (corr['SalePrice'].sort_values(ascending=False)[:5], '\n')
print (corr['SalePrice'].sort_values(ascending=False)[-5:])

plt.scatter(x=train["GrLivArea"], y=normal_dis_price)
plt.grid(True)
plt.xlabel("Sale Price")
plt.ylabel("Above grade (ground) living area square feet")
plt.show()

plt.scatter(x=train['GarageArea'], y=np.log(train.SalePrice))
plt.grid(True)
plt.xlim(-200,1600)
plt.ylabel('Sale Price')
plt.xlabel('Garage Area')
plt.show()

train = train[train["GarageArea"] < 1200]

plt.scatter(x=train['GarageArea'], y=np.log(train.SalePrice))
plt.grid(True)
plt.xlim(-200,1600)
plt.ylabel('Sale Price')
plt.xlabel('Garage Area')
plt.show()

nulls = pd.DataFrame(train.isnull().sum().sort_values(ascending=False)[:25])
nulls.columns = ["Null Count"]
nulls.index.name = "Feature"
nulls

train.drop(["PoolQC", "MiscFeature", "Alley", "Fence", "FireplaceQu", "LotFrontage"], axis=1, inplace=True)

nums_df = train.select_dtypes(include=[np.number])
nulls_nums = pd.DataFrame(nums_df.isnull().sum().sort_values(ascending=False)[:5])
nulls_nums.columns=["Null Count"]
nulls_nums.index.name = "Feature"
nulls_nums

# 填补空缺值
mean_MV = train.MasVnrArea.mean()
train.loc[(train.MasVnrArea.isnull()), 'MasVnrArea'] = mean_MV

# 去掉 GarageYrBlt 列
train.drop(["GarageYrBlt"], axis=1, inplace=True)

nums_df_ed = train.select_dtypes(include=[np.number])
nulls_nums_ed = pd.DataFrame(nums_df_ed.isnull().sum().sort_values(ascending=False)[:5])
nulls_nums_ed.columns=["Null Count"]
nulls_nums_ed.index.name = "Feature"
nulls_nums_ed

categoricals = train.select_dtypes(exclude=[np.number])
categoricals.describe()

print ("Original: \n")
print (train.Street.value_counts(), "\n")

train['enc_street'] = pd.get_dummies(train.Street, drop_first=True)
print ("Original: \n")
print (train.enc_street.value_counts())

MSZoning_piovt_max = train.pivot_table(index="MSZoning", values="SalePrice", aggfunc=np.max)
MSZoning_piovt_max.plot(kind="bar", color="blue")
plt.xlabel("MSZoning(max value)")
plt.ylabel("Sale price")
for i, item in enumerate(MSZoning_piovt_max.values):
    plt.text(i, item, "{}".format(int(item)), ha='center', va='bottom')
plt.show()

MSZoning_piovt_min = train.pivot_table(index="MSZoning", values="SalePrice", aggfunc=np.min)
MSZoning_piovt_min.plot(kind="bar", color="blue")
plt.xlabel("MSZoning(min value)")
plt.ylabel("Sale price")
for i, item in enumerate(MSZoning_piovt_min.values):
    plt.text(i, item, "{}".format(int(item)), ha='center', va='bottom')
plt.show()

MSZoning_piovt_median = train.pivot_table(index="MSZoning", values="SalePrice", aggfunc=np.median)
MSZoning_piovt_median.plot(kind="bar", color="blue")
plt.xlabel("MSZoning(median value)")
plt.ylabel("Sale price")
for i, item in enumerate(MSZoning_piovt_median.values):
    plt.text(i, item, "{}".format(int(item)), ha='center', va='bottom')
plt.show()

Neighborhood_piovt_median = train.pivot_table(index="Neighborhood", values="SalePrice", aggfunc=np.mean)
Neighborhood_piovt_median.plot(kind="bar", color="blue")
plt.xlabel("Neighborhood(median value)")
plt.ylabel("Sale price")
plt.grid(True)
# for i, item in enumerate(Neighborhood_piovt_median.values):
#     plt.text(i, item, "{}".format(int(item)), ha='center', va='bottom')
plt.show()

dict_of_non_num_variabel = {'MSZoning':['C (all)', 'RH'],
                            'Neighborhood':[ 'NoRidge', 'NridgHt', 'StoneBr'],
                            'LotShape':['Reg'],
                            'LandContour':['Low'],
                            'LotConfig':['FR3', 'CulDSac'],
                            'LandSlope':['Gtl'],
                            'Condition1':['PosA', 'PosN', 'RRNn'],
                            'Condition2':['PosA', 'PosN'],
                            'BldgType':['1Fam', 'TwnhsE'],
                            'HouseStyle':['2.5Fin', '2Story'],
                            'RoofStyle':['Shed'],
                            'RoofMatl':['WdShngl'],
                            'Exterior1st':['CemntBd', 'ImStucc', 'Stone'],
                            'Exterior2nd':['Other'],
                            'MasVnrType':['Stone'],
                            'ExterQual':['Ex'],
                            'ExterCond':['Fa', 'Po'],
                            'Foundation':['PConc'],
                            'BsmtQual':['Ex'],
                            'BsmtCond':['Po'],
                            'BsmtExposure':['No'],
                            'BsmtFinType1':['GLQ'],
                            'BsmtFinType2':['GLQ'],
                            'Heating':['Floor', 'Grav'],
                            'HeatingQC':['Po'],
                            'Electrical':['SBrkr'],
                            'KitchenQual':['Ex'],
                            'Functional':['Maj2'],
                            'PavedDrive':['N'],
                            'SaleType':['Con', 'New'],
                            'SaleCondition':['Partial']}

# 将文字变量转换为ont-hot
for k,v in dict_of_non_num_variabel.items():
    for item in v:
        train["encode_" + k] = train[k].apply(lambda x: 0 if x == item else 1 )
        
# Utilities
train['enc_Utilities'] = pd.get_dummies(train.Utilities, drop_first=True)

# CentralAir
train['enc_CentralAir'] = pd.get_dummies(train.CentralAir, drop_first=True)

data = train.select_dtypes(include=[np.number])

y = np.log(train.SalePrice)
X = data.drop(['SalePrice', 'Id'], axis=1)
del(train)

from sklearn.model_selection import train_test_split
from sklearn import ensemble
from sklearn.metrics import mean_squared_error

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=.33)
params = {'n_estimators': 500, 'max_depth': 4, 'min_samples_split': 2,
          'learning_rate': 0.01, 'loss': 'ls'}
clf = ensemble.GradientBoostingRegressor(**params)

model = clf.fit(X_train, y_train)

mse = mean_squared_error(y_test, clf.predict(X_test))
print("MSE: %.5f" % mse)

actual_values = y_test
predictions = model.predict(X_test)
plt.grid(True)
plt.scatter(predictions, actual_values, alpha=.25,color='b') #alpha helps to show overlapping data
plt.plot([11,13.5],[11,13.5],'r--')
plt.xlabel('Predicted Price')
plt.ylabel('Actual Price')
plt.title('Linear Regression Model')
plt.show()

# 生成结果集
submission = pd.DataFrame()
submission['Id'] = test.Id

test.drop(["PoolQC", "MiscFeature", "Alley", "Fence", "FireplaceQu", "LotFrontage"], axis=1, inplace=True)

test["YearBuilt"] = 2016 - test["YearBuilt"]
test["YearRemodAdd"] = 2016 - test["YearRemodAdd"]
test["YrSold"] = 2016 - test["YrSold"]
test["GarageYrBlt"] = 2016 - test["GarageYrBlt"]
# 填补空缺值
mean_MV = test.MasVnrArea.mean()
test.loc[(test.MasVnrArea.isnull()), 'MasVnrArea'] = mean_MV

# 去掉 GarageYrBlt 列
test.drop(["GarageYrBlt"], axis=1, inplace=True)

test['enc_street'] = pd.get_dummies(test.Street, drop_first=True)

for k,v in dict_of_non_num_variabel.items():
    for item in v:
        test["encode_" + k] = test[k].apply(lambda x: 0 if x == item else 1 )

# Utilities

test['enc_Utilities'] = 1

# CentralAir
test['enc_CentralAir'] = pd.get_dummies(test.CentralAir, drop_first=True)

data_test = test.select_dtypes(include=[np.number])

feats = data_test.select_dtypes(
        include=[np.number]).drop(['Id'], axis=1).interpolate()

predictions = model.predict(feats)

final_predictions = np.exp(predictions)

print ("Original predictions are: \n", predictions[:5], "\n")
print ("Final predictions are: \n", final_predictions[:5])

submission['SalePrice'] = final_predictions
submission.head()

submission.to_csv('submission1.csv', index=False)

Image("0717_result.PNG")

Image('nent_0.00255.PNG')

from sklearn.model_selection import train_test_split
from sklearn import ensemble
from sklearn.metrics import mean_squared_error

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=.33)
from sklearn.linear_model import ElasticNet

enet  = ElasticNet(alpha=0.00255)
model = enet.fit(X_train, y_train)

mse_enet = mean_squared_error(y_test, enet.predict(X_test))
print("MSE: %.5f" % mse_enet)

Image("enet_0721.PNG")

## 举例，如何将数据进行归一化
big_num_variabel = ["MSSubClass",
                    "LotArea",
                    "MasVnrArea",
                    "BsmtFinSF1",
                    "BsmtFinSF2",
                    "BsmtUnfSF",
                    "TotalBsmtSF",
                    "1stFlrSF",
                    "2ndFlrSF",
                    "LowQualFinSF",
                    "GrLivArea",
                    "GarageArea",
                    "WoodDeckSF",
                    "OpenPorchSF",
                    "EnclosedPorch"]

from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()

for item in big_num_variabel:
    scaler_of_this = scaler.fit(train[item].values.reshape(-1,1))
    train[item + "scaled"] = scaler.fit_transform(train[item].values.reshape(-1,1), scaler_of_this)

train.drop(big_num_variabel, axis=1, inplace=True)

## 结果值为(-1,1)之间的数