import pandas as pd
from pandas import DataFrame
import numpy as np
import matplotlib.pyplot as plt

data=pd.read_csv('train.csv',header=0)
data_test=pd.read_csv('test.csv',header=0)
###查看基本信息 
#查看变量中的缺失值 
data.apply(lambda x:sum(x.isnulll()),axis=0)
data.info()
data_test.info()
data.head()

#统计信息描述 只显示数值类型的 
data.describe()
data_test.describe()
###查看变量类型
data.dtypes
data_test.dtypes
###去掉不需要的行 
data=data.drop(['PassengerId','Name','Ticket'],axis=1)
data_test=data_test.drop(['Name','Ticket'],axis=1)
###处理分类变量
data['Gender']=data['Sex'].map({'female':0,'male':1}).astype(int)
data_test['Gender']=data_test['Sex'].map({'female':0,'male':1}).astype(int)


###处理age字段 
data['Age'].mean()
data['Age'].median()
data[data['Age']>60][['Sex','Age','Pclass']]#筛选数据的方式 
data[data['Age'].isnull()][['Sex','Age','Pclass']]#筛选数据的方式 
data['Age'].dropna().hist(bins=16,range=(0,80),alpha=.5)
plt.show()
##处理age字段的缺失值信息 
#根据性别和客舱级别分别求年龄的平均值 
median_ages=np.zeros((2,3))
median_ages_test=np.zeros((2,3))
for i in range(0,2):
    for j in range(0,3):
        median_ages[i,j]=data[(data['Gender']==i )& (data['Pclass']==j+1)]['Age'].dropna().median()
        median_ages_test[i,j]=data_test[(data_test['Gender']==i) &( data_test['Pclass']==j+1)]['Age'].dropna().median()

#data[(data['Gender']==0 )& (data['Pclass']==1)]['Age'].dropna().median()
data['AgeFill']=data['Age']
data_test['AgeFill']=data_test['Age']
#填充缺失值 
for i in range(0,2):
    for j in range(0,3):
        data.loc[(data.Age.isnull())&(data.Gender==i)&(data.Pclass==j+1),'AgeFill']=median_ages[i,j]
        data_test.loc[(data_test.Age.isnull())&(data_test.Gender==i)&(data_test.Pclass==j+1),'AgeFill']=median_ages_test[i,j]
#创建字段 记录Age中哪些是原有的 哪些是后来加入的
data['AgeIsNull']=pd.isnull(data.Age).astype(int)
data_test['AgeIsNull']=pd.isnull(data_test.Age).astype(int)

#按年龄将sex分成女性 男性 小孩
def get_person(passenger):
    age,sex=passenger
    return 'child' if age<16 else sex

data['Person']=data[['Age','Sex']].apply(get_person,axis=1)
data_test['Person']=data_test[['Age','Sex']].apply(get_person,axis=1)

#创建哑变量
person_dummies=pd.get_dummies(data['Person'])
person_dummies.columns=['Child','Female','Male']
person_dummies.drop(['Male'],axis=1,inplace=True)#建模需要
person_dummies_test=pd.get_dummies(data_test['Person'])
person_dummies_test.columns=['Child','Female','Male']
person_dummies_test.drop(['Male'],axis=1,inplace=True)#建模需要

data=data.join(person_dummies)
data_test=data_test.join(person_dummies_test)

###处理embarked字段 
data['Embarked'].describe()
data['Embarked'].value_counts()#这个对分类更实用！！

temp1=data['Embarked'].value_counts()
temp1.plot(kind='bar')
plt.show()
#透视图了解一下  
temp2=data.pivot_table(values='Survived',index=['Embarked'],aggfunc=lambda x:x.mean())
temp2.plot(kind='bar')
plt.show()
#创建哑变量
embark_dummies=pd.get_dummies(data['Embarked'])
embark_dummies.drop(['S'],axis=1,inplace=True)
embark_dummies_test=pd.get_dummies(data_test['Embarked'])
embark_dummies_test.drop(['S'],axis=1,inplace=True)
data=data.join(embark_dummies)
data_test=data_test.join(embark_dummies_test)

###处理Fare字段标准化
data['Fare'].fillna(data['Fare'].median(),inplace=True)
Fare_avg=data['Fare'].mean()
Fare_std=data['Fare'].std()
data['Fare']=(data['Fare']-Fare_avg)/Fare_std
data['Fare'].hist(bins=40)
data_test['Fare'].fillna(data_test['Fare'].median(),inplace=True)
Fare_avg_test=data_test['Fare'].mean()
Fare_std_test=data_test['Fare'].std()
data_test['Fare']=(data_test['Fare']-Fare_avg_test)/Fare_std_test

###处理Family字段 ,满足某个条件的进行了赋值操作
data['FamliySize']=data['SibSp']+data['Parch']
data_test['FamliySize']=data_test['SibSp']+data_test['Parch']

data['FamliySize'].loc[data['FamliySize']>0]=1
data['FamliySize'].loc[data['FamliySize']==0]=0
data['FamliySize'].value_counts()

###处理Pclass字段 
temp3=data.pivot_table(values='Survived',index=['Pclass'],aggfunc=lambda x:x.mean())
temp3.plot(kind='bar')
plt.show()
pclass_dummies=pd.get_dummies(data['Pclass'])
pclass_dummies.columns=['class_1','class_2','class_3']
pclass_dummies.drop(['class_3'],axis=1,inplace=True)
pclass_dummies_test=pd.get_dummies(data_test['Pclass'])
pclass_dummies_test.columns=['class_1','class_2','class_3']
pclass_dummies_test.drop(['class_3'],axis=1,inplace=True)

data=data.join(pclass_dummies)
data_test=data_test.join(pclass_dummies)

###敲掉多余的列 
data.drop(['Pclass','Sex','Age','SibSp','Parch','Cabin','Embarked','Gender','AgeFill','AgeIsNull','Person'],axis=1,inplace=True)
data=data.dropna()
data_test.drop(['Pclass','Sex','Age','SibSp','Parch','Cabin','Embarked','Gender','AgeFill','AgeIsNull','Person'],axis=1,inplace=True)
data_test=data_test.dropna()


###定义训练集和测试集 
X_train=data.drop('Survived',axis=1)
Y_train=data['Survived']
X_test=data_test.drop('PassengerId',axis=1).copy()

###建模分析 
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC,LinearSVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
##1.
logreg=LogisticRegression()
logreg.fit(X_train,Y_train)
Y_pred=logreg.predict(X_test)
logreg.score(X_train,Y_train)

##2.
svc=LinearSVC()
svc.fit(X_train,Y_train)
Y_pred=svc.predict(X_test)
svc.score(X_train,Y_train)

##3.
random_forest=RandomForestClassifier(n_estimators=100)#n_estimators : integer, optional (default=10)The number of trees in the forest.
random_forest.fit(X_train,Y_train)
Y_pred=random_forest.predict(X_test)
random_forest.score(X_train,Y_train)
##4.
knn=KNeighborsClassifier()
knn.fit(X_train,Y_train)
Y_pred=svc.predict(X_test)
knn.score(X_train,Y_train)
##5.
gaussian=GaussianNB()
gaussian.fit(X_train,Y_train)
Y_pred=svc.predict(X_test)
gaussian.score(X_train,Y_train)

##6.correlation analysis
coeff_df=DataFrame(data.columns.delete(0))
coeff_df.columns=['Features']
coeff_df['Coefficient Estimate']=pd.Series(logreg.coef_[0])

submission=pd.DataFrame({'passengerId':data_test['PassengerId'],'Survived':Y_pred})
submission.to_csv('titanic.csv',index=False)