Datawhale 智慧海洋建设-Task2 数据分析

此部分为智慧海洋建设竞赛的数据分析模块，通过数据分析，可以熟悉数据，为后面的特征工程做准备，欢迎大家后续多多交流。

赛题：智慧海洋建设 1 W2 J# W: X8 B

数据分析的目的:

; N' u/ I: H# z; t3 e: V0 o! _ EDA的主要价值在于熟悉整个数据集的基本情况(缺失值、异常值)，来确定所获得数据集可以用于接下来的机器学习或者深度学习使用。了解特征之间的相关性、分布，以及特征与预测值之间的关系。为进行特征工程提供理论依据。

项目地址：https://github.com/datawhalechina/team-learning-data-mining/tree/master/wisdomOcean比赛地址：https://tianchi.aliyun.com/competition/entrance/231768/introduction?spm=5176.12281957.1004.8.4ac63eafE1rwsY

2.1 学习目标

学习如何对数据集整体概况进行分析，包括数据集的基本情况(缺失值、异常值)学习了解变量之间的相互关系、变量与预测值之间的存在关系。完成相应学习打卡任务

2.2 内容介绍

数据总体了解读取数据集并了解数据集的大小，原始特征维度；通过info了解数据类型；粗略查看数据集中各特征的基本统计量缺失值和唯一值查看数据缺失值情况查看唯一值情况

数据特性和特征分布

7 G- X- e- W% d/ p7 z G

三类渔船轨迹的可视化坐标序列可视化三类渔船速度和方向序列可视化三类渔船速度和方向的数据分布

作业一：剔除异常点后画图

import pandas as pd

import geopandas as gpd

4 j3 a# z/ O' e7 \" e6 S0 a, l

from pyproj import Proj

! \' q5 p$ l$ D

from keplergl import KeplerGl

4 ~" R- x( D' @( m

from tqdm import tqdm

import os

' |2 h' V; A( ^3 h

import matplotlib.pyplot as plt

% L5 z( O; C+ S ?* _

import shapely

* Y% `, ]$ y* a$ r$ l

import numpy as np

4 P2 w+ U3 A$ G' T

from datetime import datetime

: v1 s Y$ { O

import warnings

warnings.filterwarnings(ignore)

plt.rcParams[font.sans-serif] = [SimSun] # 指定默认字体为新宋体。

plt.rcParams[axes.unicode_minus] = False # 解决保存图像时 负号- 显示为方块和报错的问题。

#获取文件夹中的数据

+ M; M8 n5 E f% y/ `$ G2 R

def get_data(file_path,model):

; t/ y/ V- {9 t$ I

assert model in [train, test], {} Not Support this type of file.format(model)

) k1 t* t0 ^# F5 {

paths = os.listdir(file_path)

$ w1 w8 B* e( S' {% F

# print(len(paths))

tmp = []

for t in tqdm(range(len(paths))):

p = paths[t]

, O$ Y" Q7 W, k7 `( \

with open({}/{}.format(file_path, p), encoding=utf-8) as f:

next(f)

- A5 N3 M/ P9 v$ t" o) A, L7 r3 x

for line in f.readlines():

tmp.append(line.strip().split(,))

2 D/ T& ^0 a/ g0 n7 j5 b

tmp_df = pd.DataFrame(tmp)

5 f& l) D9 s3 n9 x

if model == train:

: k3 o! r1 C) w* i

tmp_df.columns = [ID, lat, lon, speed, direction, time, type]

$ e$ y4 L9 t$ @* l

else:

tmp_df[type] = unknown

tmp_df.columns = [ID, lat, lon, speed, direction, time, type]

9 w. W$ J3 O1 o. c0 f- Y

tmp_df[lat] = tmp_df[lat].astype(float)

L; z- p9 [$ m H* E. X8 p! }

tmp_df[lon] = tmp_df[lon].astype(float)

. S+ r1 ?0 I' M; B0 ~- m

tmp_df[speed] = tmp_df[speed].astype(float)

tmp_df[direction] = tmp_df[direction].astype(int)#如果该行代码运行失败，请尝试更新pandas的版本

$ a& }5 o1 H. w0 y

return tmp_df

# 平面坐标转经纬度，供初赛数据使用

# 选择标准为NAD83 / California zone 6 (ftUS) (EPSG:2230)，查询链接：CS2CS - Transform Coordinates On-line - MyGeodata Cloud

+ [% V! @) u/ v& t3 L) Z

def transform_xy2lonlat(df):

6 ^5 k' [: H \" x' V6 F

x = df[lat].values

4 j' p0 s o3 L r8 x/ h

y = df[lon].values

/ U4 V1 B) x4 z6 O- o( ^2 r$ n

p=Proj(+proj=lcc +lat_1=33.88333333333333 +lat_2=32.78333333333333 +lat_0=32.16666666666666 +lon_0=-116.25 +x_0=2000000.0001016 +y_0=500000.0001016001 +datum=NAD83 +units=us-ft +no_defs )

df[lon], df[lat] = p(y, x, inverse=True)

return df

#修改数据的时间格式

8 t, r" E9 V3 X& f' @8 @

def reformat_strtime(time_str=None, START_YEAR="2019"):

; E2 L$ Q2 ]/ G0 y& e$ d4 I

"""Reformat the strtime with the form 08 14 to START_YEAR-08-14 """

time_str_split = time_str.split(" ")

2 \; N" h1 ~ l: u9 M1 m

time_str_reformat = START_YEAR + "-" + time_str_split[0][:2] + "-" + time_str_split[0][2:4]

- j7 L8 @6 E, A2 H9 j f W

time_str_reformat = time_str_reformat + " " + time_str_split[1]

# time_reformat=datetime.strptime(time_str_reformat,%Y-%m-%d %H:%M:%S)

' Q B: ?$ P( n7 t9 O+ N+ W, n# K

return time_str_reformat

#计算两个点的距离

def haversine_np(lon1, lat1, lon2, lat2):

7 p5 }+ K# I% U$ a; w

lon1, lat1, lon2, lat2 = map(np.radians, [lon1, lat1, lon2, lat2])

( m4 ?, l& ? i& g& T- m

dlon = lon2 - lon1

3 y8 a# ~3 N b; x3 I. G9 Y% S" B

dlat = lat2 - lat1

a = np.sin(dlat/2.0)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2.0)**2

c = 2 * np.arcsin(np.sqrt(a))

4 k/ A4 p2 M2 F+ n3 J! i

km = 6367 * c

2 \3 ~9 z& u1 M

return km * 1000

$ H3 `- a; O: }

def compute_traj_diff_time_distance(traj=None):

"""Compute the sampling time and the coordinate distance."""

# 计算时间的差值

5 A0 s y- @ U* G& c* ?7 p; x+ Z

time_diff_array = (traj["time"].iloc[1:].reset_index(drop=True) - traj[

"time"].iloc[:-1].reset_index(drop=True)).dt.total_seconds() / 60

# 计算坐标之间的距离

& `$ i- S' Q$ h: Q5 d- u, c( ^# t% z

dist_diff_array = haversine_np(traj["lon"].values[1:], # lon_0

traj["lat"].values[1:], # lat_0

traj["lon"].values[:-1], # lon_1

traj["lat"].values[:-1] # lat_1

* L, |- S6 |6 v2 o- i

)

! e W3 A% H& w& q

# 填充第一个值

time_diff_array = [time_diff_array.mean()] + time_diff_array.tolist()

; g) f4 A( d/ b" h5 i2 e

dist_diff_array = [dist_diff_array.mean()] + dist_diff_array.tolist()

+ h; ^. g$ ^& i4 R- P& t. L

traj.loc[list(traj.index),time_array] = time_diff_array

7 y9 p- X6 \2 i: {, y! Z

traj.loc[list(traj.index),dist_array] = dist_diff_array

return traj

7 N( ~7 u) c3 @

#对轨迹进行异常点的剔除

def assign_traj_anomaly_points_nan(traj=None, speed_maximum=23,

time_interval_maximum=200,

6 P2 m' [2 H% T) s: A& P' f1 `: u

coord_speed_maximum=700):

"""Assign the anomaly points in traj to np.nan."""

5 A) M4 Y# ]$ y# t j: _* C

def thigma_data(data_y,n):

data_x =[i for i in range(len(data_y))]

ymean = np.mean(data_y)

ystd = np.std(data_y)

9 E Y1 m p0 t K$ V2 M5 D8 n) h

threshold1 = ymean - n * ystd

threshold2 = ymean + n * ystd

' V% h$ N. F4 v( `- }# F* B

judge=[]

for data in data_y:

if (data < threshold1)|(data> threshold2):

judge.append(True)

else:

judge.append(False)

( i: A" K& {& i2 W5 C6 n

return judge

3 ~! m7 u. e# }

# Step 1: The speed anomaly repairing

is_speed_anomaly = (traj["speed"] > speed_maximum) | (traj["speed"] < 0)

. w' r; `6 S* ]% g4 f0 F: a

traj["speed"][is_speed_anomaly] = np.nan

* b! |3 o7 \, D7 ]; u: ?8 u5 s

# Step 2: 根据距离和时间计算速度

9 Q- A" |! |: S2 C( {8 S) \

is_anomaly = np.array([False] * len(traj))

# a m5 q' E4 V! C! u8 p: g

traj["coord_speed"] = traj["dist_array"] / traj["time_array"]

4 A8 z6 [% F+ f' ~0 D3 W

# Condition 1: 根据3-sigma算法剔除coord speed以及较大时间间隔的点

M" S5 I+ A1 y s

is_anomaly_tmp = pd.Series(thigma_data(traj["time_array"],3)) | pd.Series(thigma_data(traj["coord_speed"],3))

is_anomaly = is_anomaly | is_anomaly_tmp

is_anomaly.index=traj.index

$ D0 |9 k: {7 O. }

# Condition 2: 轨迹点的3-sigma异常处理

+ p }8 w2 }1 z& B

traj = traj[~is_anomaly].reset_index(drop=True)

: _# x: @, R3 R8 i2 C; A2 A

is_anomaly = np.array([False] * len(traj))

6 \. m5 F* h! m" v# x* V

if len(traj) != 0:

lon_std, lon_mean = traj["lon"].std(), traj["lon"].mean()

lat_std, lat_mean = traj["lat"].std(), traj["lat"].mean()

( |9 R8 j s- H/ z; a4 O4 `; j* g8 A+ k

lon_low, lon_high = lon_mean - 3 * lon_std, lon_mean + 3 * lon_std

lat_low, lat_high = lat_mean - 3 * lat_std, lat_mean + 3 * lat_std

; {0 W) b1 L& v9 d7 p; x

is_anomaly = is_anomaly | (traj["lon"] > lon_high) | ((traj["lon"] < lon_low))

- E- I/ i+ `4 U

is_anomaly = is_anomaly | (traj["lat"] > lat_high) | ((traj["lat"] < lat_low))

0 Z& t) n9 S6 L; }2 h% T

traj = traj[~is_anomaly].reset_index(drop=True)

return traj, [len(is_speed_anomaly) - len(traj)]

- A- p9 B) r3 [6 E- Q$ B$ x+ F

df=get_data(rC:\Users\admin\hy_round1_train_20200102,train)

! I) W" U& a' U {3 Y2 d6 N+ U/ u

#对轨迹进行异常点剔除，对nan值进行线性插值

ID_list=list(pd.DataFrame(df[ID].value_counts()).index)

DF_NEW=[]

& h1 Z5 A6 h/ e4 q7 c9 ~

Anomaly_count=[]

for ID in tqdm(ID_list):

% ?' D; w1 }7 H! q; g3 u4 ]* T/ C

df_id=compute_traj_diff_time_distance(df[df[ID]==ID])

df_new,count=assign_traj_anomaly_points_nan(df_id)

% X" ^$ \$ ~: l- @& }

df_new["speed"] = df_new["speed"].interpolate(method="linear", axis=0)

df_new = df_new.fillna(method="bfill")

+ _. v7 |! H, x+ b( m, k. E

df_new = df_new.fillna(method="ffill")

7 K; [( Z; q) P, o" a8 o

df_new["speed"] = df_new["speed"].clip(0, 23)

" f) C- `* q1 C

Anomaly_count.append(count)#统计每个id异常点的数量有多少

DF_NEW.append(df_new)

2 _: Y, Q$ t" N* \- M% w4 Z5 t

#将数据写入到pkl格式

2 e9 g$ v" L: y. Y

load_save = Load_Save_Data()

load_save.save_data(DF_NEW,"C:/Users/admin/wisdomOcean/data_tmp1/total_data.pkl")

#### 三类渔船速度和方向可视化

# 把训练集的所有数据,根据类别存放到不同的数据文件中

, S" w" o/ i& r% \" F! n1 U

def get_diff_data():

Path = "C:/Users/admin/wisdomOcean/data_tmp1/total_data.pkl"

with open(Path,"rb") as f:

( j3 v6 X5 E1 ]* r: [

total_data = pickle.load(f)

0 Q& W/ V x1 ]! M- H# c1 Z

load_save = Load_Save_Data()

kind_data = ["刺网","围网","拖网"]

) H0 E- B- {# P; x; k2 a3 T

file_names = ["ciwang_data.pkl","weiwang_data.pkl","tuowang_data.pkl"]

+ J, a6 K) W" w* Z' R% e$ Y! _, \, Y9 s

for i,datax in enumerate(kind_data):

data_type = [data for data in total_data if data["type"].unique()[0] == datax]

- x0 I9 }, S+ u: Q! I: l3 n

load_save.save_data(data_type,"C:/Users/admin/wisdomOcean/data_tmp1/" + file_names[i])

$ S o' k, N. C* g' Z# ]) _7 ~6 M

get_diff_data()

#对轨迹进行异常点剔除，对nan值进行线性插值

; M: s- j4 k2 K$ S5 v0 o$ _' `

ID_list=list(pd.DataFrame(df[ID].value_counts()).index)

DF_NEW=[]

Anomaly_count=[]

b+ G5 d% |4 _' H! _, G# w

for ID in tqdm(ID_list):

df_id=compute_traj_diff_time_distance(df[df[ID]==ID])

df_new,count=assign_traj_anomaly_points_nan(df_id)

* p* U! C% @/ q" B4 s3 z g

df_new["speed"] = df_new["speed"].interpolate(method="linear", axis=0)

7 E$ x0 h- S" `0 u6 L- Q9 v

df_new = df_new.fillna(method="bfill")

df_new = df_new.fillna(method="ffill")

5 ~1 E: n# _, z0 n) L

df_new["speed"] = df_new["speed"].clip(0, 23)

Anomaly_count.append(count)#统计每个id异常点的数量有多少

( ?- L; ^5 K, ^4 W6 X

DF_NEW.append(df_new)

5 H r$ k$ [$ z: W/ \

# 每类轨迹，随机选取某个渔船，可视化速度序列和方向序列

8 d2 d' e6 n& g6 a* H

def visualize_three_traj_speed_direction():

4 Z4 ^8 W. d( w/ P: x+ N2 C& R

fig,axes = plt.subplots(nrows=3,ncols=2,figsize=(20,15))

plt.subplots_adjust(wspace=0.3,hspace=0.3)

# 随机选出刺网的三条轨迹进行可视化

% \! \" P8 x6 I8 X( c

file_types = ["ciwang_data","weiwang_data","tuowang_data"]

' D- a5 ]! [' n h2 r/ T# Z* h

speed_types = ["ciwang_speed","weiwang_speed","tuowang_speed"]

doirections = ["ciwang_direction","weiwang_direction","tuowang_direction"]

colors = [pink, lightblue, lightgreen]

. N! f {, u( ]

for i,file_name in tqdm(enumerate(file_types)):

datax = get_random_one_traj(type=file_name)

# f d2 U @7 O5 |2 |

x_data = datax["速度"].loc[-1:].values

9 o' d$ C$ v0 |5 S( M

y_data = datax["方向"].loc[-1:].values

* b1 B& Y" O. ]5 p) y9 e- n

axes[i][0].plot(range(len(x_data)), x_data, label=speed_types[i], color=colors[i])

: _# x R5 m. c/ P; l* N; t

axes[i][0].grid(alpha=2)

% N7 Z0 {9 [, V/ x( ?5 I& ?

axes[i][0].legend(loc="best")

/ K1 x; _' Q" z% a ]

axes[i][1].plot(range(len(y_data)), y_data, label=doirections[i], color=colors[i])

' `" z. U) K z( g; t* T/ ~3 Q2 J4 u3 B

axes[i][1].grid(alpha=2)

axes[i][1].legend(loc="best")

plt.show()

visualize_three_traj_speed_direction()

* j: u) Z2 @. e8 W8 X5 j% g

0 ], K3 q; S" c0 y' X' y5 ~

作业二：相关性分析。

data_train.loc[data_train[type]==刺网,type_id]=1

data_train.loc[data_train[type]==围网,type_id]=2

; r: `- X6 X, m* }) ]- D* M; M$ l

data_train.loc[data_train[type]==拖网,type_id]=3

: @8 d4 |* W; m) F' B' w

f, ax = plt.subplots(figsize=(9, 6))

3 j: e' j1 E! z: J) i. z) w, \4 O ~

ax = sns.heatmap(np.abs(df.corr()),annot=True)

plt.show()

从图中可以清楚看到，经纬度和速度跟类型相关性比较大。

+ D! y$ y9 p! V5 G, V; L8 s/ X