包导入
import numpy as np
import pandas as pd
基本介绍
pandas序列
s = pd.Series([1,3,2,np.pi,0])
>s
>
0 1.000000
1 3.000000
2 2.000000
3 3.141593
4 0.000000
dtype: float64
DataFrame
dates = pd.date_range('20200310',periods=5)
>dates
>
DatetimeIndex(['2020-03-10', '2020-03-11', '2020-03-12', '2020-03-13',
'2020-03-14'],
dtype='datetime64[ns]', freq='D')
方法一:
df1 = pd.DataFrame(np.random.randn(5,4),index = dates,columns = ['A','B','C','D'])
df1
A B C D
2020-03-10 -0.905463 1.347745 -0.543910 1.893159
2020-03-11 1.259007 0.703927 -0.699856 1.115103
2020-03-12 0.273644 -0.433317 0.933585 0.275534
2020-03-13 -0.041525 0.322617 0.329219 1.782954
2020-03-14 -0.461506 0.452861 -0.795465 1.214859
方法二:
df2 = pd.DataFrame(np.arange(12).reshape((3,4)))
df2
0 1 2 3
0 0 1 2 3
1 4 5 6 7
2 8 9 10 11
方法三:(字典方式)
df3 = pd.DataFrame({'A':9.,
'B':pd.Timestamp('20200310'),
'C':pd.Series(1,index=list(range(4)),dtype='float32'),
'D':np.array([2]*4,dtype='int32'),
'E':pd.Categorical(['flower','pen','flower','pen']),
'F':'foo'}
>df3
A B C D E F
0 9.0 2020-03-10 1.0 2 flower foo
1 9.0 2020-03-10 1.0 2 pen foo
2 9.0 2020-03-10 1.0 2 flower foo
3 9.0 2020-03-10 1.0 2 pen foo
相关属性:
>df3.dtypes # 获取DataFrame元素属性
A float64
B datetime64[ns]
C float32
D int32
E category
F object
dtype: object
>df2.columns # 获取DataFrame列索引名称
RangeIndex(start=0, stop=4, step=1)
>df3.columns
Index(['A', 'B', 'C', 'D', 'E', 'F'], dtype='object')
>df3.values # 获取DataFrame元素值
array([[9.0, Timestamp('2020-03-10 00:00:00'), 1.0, 2, 'flower', 'foo'],
[9.0, Timestamp('2020-03-10 00:00:00'), 1.0, 2, 'pen', 'foo'],
[9.0, Timestamp('2020-03-10 00:00:00'), 1.0, 2, 'flower', 'foo'],
[9.0, Timestamp('2020-03-10 00:00:00'), 1.0, 2, 'pen', 'foo']],
dtype=object)
>df3.describe() # 获取DataFrame属性
A C D
count 4.0 4.0 4.0
mean 9.0 1.0 2.0
std 0.0 0.0 0.0
min 9.0 1.0 2.0
25% 9.0 1.0 2.0
50% 9.0 1.0 2.0
75% 9.0 1.0 2.0
max 9.0 1.0 2.0
>df3.T # DataFrame转置(类似矩阵)
0 ... 3
A 9 ... 9
B 2020-03-10 00:00:00 ... 2020-03-10 00:00:00
C 1 ... 1
D 2 ... 2
E flower ... pen
F foo ... foo
[6 rows x 4 columns]
>df3.sort_index(axis=1,ascending=False) # 获取DataFrame按索引排序
F E D C B A
0 foo flower 2 1.0 2020-03-10 9.0
1 foo pen 2 1.0 2020-03-10 9.0
2 foo flower 2 1.0 2020-03-10 9.0
3 foo pen 2 1.0 2020-03-10 9.0
>df3.sort_values(by='E')
# 获取DataFrame按某一列的值排序
A B C D E F
0 9.0 2020-03-10 1.0 2 flower foo
2 9.0 2020-03-10 1.0 2 flower foo
1 9.0 2020-03-10 1.0 2 pen foo
3 9.0 2020-03-10 1.0 2 pen foo
Pandas数据选择
dates = pd.date_range('20200310',periods=6)
df = pd.DataFrame(np.arange(24).reshape((6,4)),index = dates,columns = ['A','B','C','D'])
>df
A B C D
2020-03-10 0 1 2 3
2020-03-11 4 5 6 7
2020-03-12 8 9 10 11
2020-03-13 12 13 14 15
2020-03-14 16 17 18 19
2020-03-15 20 21 22 23
方法一:(最一般方法)
print(df['A'],df.A)
2020-03-10 0
2020-03-11 4
2020-03-12 8
2020-03-13 12
2020-03-14 16
2020-03-15 20
Freq: D, Name: A, dtype: int64 2020-03-10 0
2020-03-11 4
2020-03-12 8
2020-03-13 12
2020-03-14 16
2020-03-15 20
Freq: D, Name: A, dtype: int64
print(df[0:3],df['20200313':'20200315'])
A B C D
2020-03-10 0 1 2 3
2020-03-11 4 5 6 7
2020-03-12 8 9 10 11 A B C D
2020-03-13 12 13 14 15
2020-03-14 16 17 18 19
2020-03-15 20 21 22 23
方法二:(以标签选择)
print(df.loc['20200311']) # 行标签
A 4
B 5
C 6
D 7
Name: 2020-03-11 00:00:00, dtype: int64
print(df.loc[:,['A','C']]) # 列标签
A C
2020-03-10 0 2
2020-03-11 4 6
2020-03-12 8 10
2020-03-13 12 14
2020-03-14 16 18
2020-03-15 20 22
print(df.loc[‘20200310,['A','C']])
方法三:(以位置
print(df.iloc[3]) #第三行
print(df.iloc[3,1]) #第三行第一位
print(df.iloc[3:5,1:3])
print(df.iloc[3:5,1])
2020-03-13 13
2020-03-14 17
Freq: D, Name: B, dtype: int64
方法四:混合方法
print(df.ix[:4,['A','D']]) # 行以数字筛选,列以标签筛选
方法五: Boolean indexing
print(df[df.A > 8])
Pandas设置值
df.iloc[2,3] = 102 # 把第二行第三列的值改成102
df.loc['20200310',3] = 132
df[df.A>4]=0 # 后面的数字会被更改
df.A[df.A>4]=0 # 后面的数字不会被更改,只改A那一列的
df.B[df.A>4]=0 # 后面的数字不会被更改,只改B那一列的
df['F'] = np.nan #在DataFrame中定义新的一个空列
Pandas处理丢失数据
dates = pd.date_range('20200310',periods=6)
df = pd.DataFrame(np.arange(24).reshape((6,4)),index = dates,columns = ['A','B','C','D'])
df.iloc[0,1] = np.nan
df.iloc[1,2] = np.nan
>df
A B C D
2020-03-10 0 NaN 2.0 3
2020-03-11 4 5.0 NaN 7
2020-03-12 8 9.0 10.0 11
2020-03-13 12 13.0 14.0 15
2020-03-14 16 17.0 18.0 19
2020-03-15 20 21.0 22.0 23
print(df.dropna(axis=0,how='any')) #按行丢掉含空值的 # how = {'any','all}
A B C D
2020-03-12 8 9.0 10.0 11
2020-03-13 12 13.0 14.0 15
2020-03-14 16 17.0 18.0 19
2020-03-15 20 21.0 22.0 23
print(df.fillna(value=0)) # 填充
A B C D
2020-03-10 0 0.0 2.0 3
2020-03-11 4 5.0 0.0 7
2020-03-12 8 9.0 10.0 11
2020-03-13 12 13.0 14.0 15
2020-03-14 16 17.0 18.0 19
2020-03-15 20 21.0 22.0 23
print(df.isnull()) # 检查是否有缺少
A B C D
2020-03-10 False True False False
2020-03-11 False False True False
2020-03-12 False False False False
2020-03-13 False False False False
2020-03-14 False False False False
2020-03-15 False False False False
print(np.any(df.isnull()) == True)
True
Pandas导入导出
导入:以CSV文件为例
data = pd.read_csv('filename')# pandas会自动添加相应索引
print(data)
导出
data.to_pickle('filename.pickle') # 如存为pickle文件
Pandas合并concat
# concatenating
# ignore index
df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['a','b','c','d'])
df3 = pd.DataFrame(np.ones((3,4))*2, columns=['a','b','c','d'])
res = pd.concat([df1, df2, df3], axis=0, ignore_index=True)
# join, ('inner', 'outer')
df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d'], index=[1,2,3])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['b','c','d', 'e'], index=[2,3,4])
res = pd.concat([df1, df2], axis=1, join='outer',ignore_index=True) # 默认模式,自动填充空值
res = pd.concat([df1, df2], axis=1, join='inner',ignore_index=True) #裁剪对齐
# join_axes
res = pd.concat([df1, df2], axis=1, join_axes=[df1.index]) # 按第一个的行索引进行合并,其余裁剪
# append 上下合并
df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['b','c','d', 'e'], index=[2,3,4])
res = df1.append(df2, ignore_index=True)
res = df1.append([df2, df3])
s1 = pd.Series([1,2,3,4], index=['a','b','c','d']) # 一行一行地添加
res = df1.append(s1, ignore_index=True)
print(res)
注:concat适用于行合并、列合并;join适用于左右合并;append上下合并;
Pandas合并merge
# merging two df by key/keys
left = pd.DataFrame({'key': ['K0', 'K1', 'K2', 'K3'],
... 'A': ['A0', 'A1', 'A2', 'A3'],
... 'B': ['B0', 'B1', 'B2', 'B3']})
... right = pd.DataFrame({'key': ['K0', 'K1', 'K2', 'K3'],
... 'C': ['C0', 'C1', 'C2', 'C3'],
... 'D': ['D0', 'D1', 'D2', 'D3']})
print(left)
key A B
0 K0 A0 B0
1 K1 A1 B1
2 K2 A2 B2
3 K3 A3 B3
print(right)
key C D
0 K0 C0 D0
1 K1 C1 D1
2 K2 C2 D2
3 K3 C3 D3
res = pd.merge(left, right, on='key')
print(res)
key A B C D
0 K0 A0 B0 C0 D0
1 K1 A1 B1 C1 D1
2 K2 A2 B2 C2 D2
3 K3 A3 B3 C3 D3
# consider two keys
left = pd.DataFrame({'key1': ['K0', 'K0', 'K1', 'K2'],
'key2': ['K0', 'K1', 'K0', 'K1'],
'A': ['A0', 'A1', 'A2', 'A3'],
'B': ['B0', 'B1', 'B2', 'B3']})
right = pd.DataFrame({'key1': ['K0', 'K1', 'K1', 'K2'],
'key2': ['K0', 'K0', 'K0', 'K0'],
'C': ['C0', 'C1', 'C2', 'C3'],
'D': ['D0', 'D1', 'D2', 'D3']})
print(left)
key1 key2 A B
0 K0 K0 A0 B0
1 K0 K1 A1 B1
2 K1 K0 A2 B2
3 K2 K1 A3 B3
print(right)
key1 key2 C D
0 K0 K0 C0 D0
1 K1 K0 C1 D1
2 K1 K0 C2 D2
3 K2 K0 C3 D3
res = pd.merge(left, right, on=['key1', 'key2'], how='inner') # default for how='inner'
print(res)
key1 key2 A B C D
0 K0 K0 A0 B0 C0 D0
1 K1 K0 A2 B2 C1 D1
2 K1 K0 A2 B2 C2 D2
# how = ['left', 'right', 'outer', 'inner']
res = pd.merge(left, right, on=['key1', 'key2'], how='left')
print(res)
key1 key2 A B C D
0 K0 K0 A0 B0 C0 D0
1 K0 K1 A1 B1 NaN NaN
2 K1 K0 A2 B2 C1 D1
3 K1 K0 A2 B2 C2 D2
4 K2 K1 A3 B3 NaN NaN
res = pd.merge(left, right, on=['key1', 'key2'], how='right')
print(res)
key1 key2 A B C D
0 K0 K0 A0 B0 C0 D0
1 K1 K0 A2 B2 C1 D1
2 K1 K0 A2 B2 C2 D2
3 K2 K0 NaN NaN C3 D3
df1 = pd.DataFrame({'col1':[0,1], 'col_left':['a','b']})
... df2 = pd.DataFrame({'col1':[1,2,2],'col_right':[2,2,2]})
print(df1)
col1 col_left
0 0 a
1 1 b
print(df2)
col1 col_right
0 1 2
1 2 2
2 2 2
res = pd.merge(df1, df2, on='col1', how='outer', indicator=True)
print(res)
col1 col_left col_right _merge
0 0 a NaN left_only
1 1 b 2.0 both
2 2 NaN 2.0 right_only
3 2 NaN 2.0 right_only
# give the indicator a custom name
res = pd.merge(df1, df2, on='col1', how='outer', indicator='indicator_column')
print(res)
col1 col_left col_right indicator_column
0 0 a NaN left_only
1 1 b 2.0 both
2 2 NaN 2.0 right_only
3 2 NaN 2.0 right_only
# merged by index
left = pd.DataFrame({'A': ['A0', 'A1', 'A2'],
'B': ['B0', 'B1', 'B2']},
index=['K0', 'K1', 'K2'])
right = pd.DataFrame({'C': ['C0', 'C2', 'C3'],
'D': ['D0', 'D2', 'D3']},
index=['K0', 'K2', 'K3'])
print(left)
A B
K0 A0 B0
K1 A1 B1
K2 A2 B2
print(right)
C D
K0 C0 D0
K2 C2 D2
K3 C3 D3
# left_index and right_index
res = pd.merge(left, right, left_index=True, right_index=True, how='outer')
print(res)
A B C D
K0 A0 B0 C0 D0
K1 A1 B1 NaN NaN
K2 A2 B2 C2 D2
K3 NaN NaN C3 D3
# handle overlapping
boys = pd.DataFrame({'k': ['K0', 'K1', 'K2'], 'age': [1, 2, 3]})
girls = pd.DataFrame({'k': ['K0', 'K0', 'K3'], 'age': [4, 5, 6]})
print(boys)
k age
0 K0 1
1 K1 2
2 K2 3
print(girls)
k age
0 K0 4
1 K0 5
2 K3 6
res = pd.merge(boys, girls, on='k', suffixes=['_boy', '_girl'], how='inner')
print(res)
print(res)
k age_boy age_girl
0 K0 1 4
1 K0 1 5
Pandas plot 画图
import matplotlib.pyplot as plt
# Series
data = pd.Series(np.random.randn(1000), index=np.arange(1000))
data = data.cumsum() # 累加的过程
data.plot()
plt.show()
# DataFrame
data = pd.DataFrame(np.random.randn(1000, 4), index=np.arange(1000), columns=list("ABCD"))
data = data.cumsum()
data.plot()
plt.show()
# plot methods:
# 'bar', 'hist', 'box', 'kde', 'area', scatter', hexbin', 'pie'
ax = data.plot.scatter(x='A', y='B', color='DarkBlue', label="Class 1")
data.plot.scatter(x='A', y='C', color='LightGreen', label='Class 2', ax=ax)
plt.show()