四種將序列數據轉換爲Pandas DataFrame數據方法效率對比

時間 2019-11-30

標籤四種序列數據轉換 pandas dataframe 方法效率對比欄目 Spark 简体版

原文原文鏈接

該測試程序是將protocol變量轉換爲DataFrame類型爲了便於使用numpy等數學工具
ImuEncode_pb2 是使用protocol生成的modelpython

這是我使用的一些變量的聲明app

import time 
import ImuEncode_pb2
import numpy as np
import pandas as pd 
from pandas.core.frame import DataFrame
from ImuSignal import LegString
from ImuSignal import LegPositionString
from ImuSignal import SignalAttributeDict
from OnlineClassifier import OnlineClassifier

ImuSignalDataset = ImuEncode_pb2.ImuSignalDataset()
for i in range(50):
    imusignal =ImuSignalDataset.imusignal.add()
    for j in range(4):
        signal = imusignal.signal.add()
        signal.TimeStamp = 0
        signal.acc.AccX = 0
        signal.acc.AccY = 0
        signal.acc.AccZ = 0
        signal.angle.Roll = 0
        signal.angle.Pitch = 0
        signal.anglerate.Course = 0
        signal.quaternion.v = 0
        signal.quaternion.x = 0
        signal.quaternion.y = 0
        signal.quaternion.z = 0
        signal.leg = ImuEncode_pb2.Signal.RT
        signal.legposition = ImuEncode_pb2.Signal.Foot

rate = 100 #sample rate
period = 0.5 #classify period
Leg = ['RT','LT']
Leg_Position = ['Foot','Thigh']
angle = ['Roll','Pitch','Course']
accel = ['AccX','AccY','AccZ']

特別注意下面代碼段的加入，這是對於內存的聲明，我是隨便賦值不知道Python變量的聲明原理後續完善工具

SignalFrame = DataFrame()
for leg in Leg:
    for leg_position in  Leg_Position:
        for signal in  accel + angle:
            SignalFrame[leg_position+"_"+signal+"_"+leg]  = np.arange(999999999,999999999+int(period*rate),1)/1000000

該方法不簡單而且耗時應該是最長的，測試耗時爲： 0.09793972969055176測試

SignalFrame = DataFrame()
for leg in Leg:
    for leg_position in  Leg_Position:
        for signal in  accel + angle:
            SignalFrame[leg_position+"_"+signal+"_"+leg]  = np.arange(999999999,999999999+int(period*rate),1)/1000000

starttime = time.time()
SignalFrame = pd.concat([pd.DataFrame(data={"Foot_AccX_RT":imusignal.signal[0].acc.AccX,  "Foot_AccY_RT":imusignal.signal[0].acc.AccY,        "Foot_AccZ_RT":imusignal.signal[0].acc.AccZ,\
            "Foot_Roll_RT":imusignal.signal[0].angle.Roll,  "Foot_Pitch_RT":imusignal.signal[0].angle.Pitch,    "Foot_Course_RT":imusignal.signal[0].anglerate.Course,\
            "Thigh_AccX_RT":imusignal.signal[1].acc.AccX,   "Thigh_AccY_RT":imusignal.signal[1].acc.AccY,       "Thigh_AccZ_RT":imusignal.signal[1].acc.AccZ,\
            "Thigh_Roll_RT":imusignal.signal[1].angle.Roll, "Thigh_Pitch_RT":imusignal.signal[1].angle.Pitch,   "Thigh_Course_RT":imusignal.signal[1].anglerate.Course,\
            "Foot_AccX_LT":imusignal.signal[2].acc.AccX,    "Foot_AccY_LT":imusignal.signal[2].acc.AccY,        "Foot_AccZ_LT":imusignal.signal[2].acc.AccZ,\
            "Foot_Roll_LT":imusignal.signal[2].angle.Roll,  "Foot_Pitch_LT":imusignal.signal[2].angle.Pitch,    "Foot_Course_LT":imusignal.signal[2].anglerate.Course,\
            "Thigh_AccX_LT":imusignal.signal[3].acc.AccX,   "Thigh_AccY_LT":imusignal.signal[3].acc.AccY,       "Thigh_AccZ_LT":imusignal.signal[3].acc.AccZ,\
            "Thigh_Roll_LT":imusignal.signal[3].angle.Roll, "Thigh_Pitch_LT":imusignal.signal[3].angle.Pitch,   "Thigh_Course_LT":imusignal.signal[3].anglerate.Course,\
        },index=[0]) for imusignal in ImuSignalDataset.imusignal],ignore_index=True)
endtime = time.time()
print("cost time:",endtime-starttime)

該方法不簡單而且耗時長，測試耗時爲： 0.04797029495239258ui

SignalFrame = DataFrame()

for leg in Leg:
    for leg_position in  Leg_Position:
        for signal in  accel + angle:
            SignalFrame[leg_position+"_"+signal+"_"+leg]  = np.arange(999999999,999999999+int(period*rate),1)/1000000

starttime = time.time()
for imusignal in ImuSignalDataset.imusignal:
    SignalFrame = SignalFrame.append({"Foot_AccX_RT":imusignal.signal[0].acc.AccX,  "Foot_AccY_RT":imusignal.signal[0].acc.AccY,        "Foot_AccZ_RT":imusignal.signal[0].acc.AccZ,\
            "Foot_Roll_RT":imusignal.signal[0].angle.Roll,  "Foot_Pitch_RT":imusignal.signal[0].angle.Pitch,    "Foot_Course_RT":imusignal.signal[0].anglerate.Course,\
            "Thigh_AccX_RT":imusignal.signal[1].acc.AccX,   "Thigh_AccY_RT":imusignal.signal[1].acc.AccY,       "Thigh_AccZ_RT":imusignal.signal[1].acc.AccZ,\
            "Thigh_Roll_RT":imusignal.signal[1].angle.Roll, "Thigh_Pitch_RT":imusignal.signal[1].angle.Pitch,   "Thigh_Course_RT":imusignal.signal[1].anglerate.Course,\
            "Foot_AccX_LT":imusignal.signal[2].acc.AccX,    "Foot_AccY_LT":imusignal.signal[2].acc.AccY,        "Foot_AccZ_LT":imusignal.signal[2].acc.AccZ,\
            "Foot_Roll_LT":imusignal.signal[2].angle.Roll,  "Foot_Pitch_LT":imusignal.signal[2].angle.Pitch,    "Foot_Course_LT":imusignal.signal[2].anglerate.Course,\
            "Thigh_AccX_LT":imusignal.signal[3].acc.AccX,   "Thigh_AccY_LT":imusignal.signal[3].acc.AccY,       "Thigh_AccZ_LT":imusignal.signal[3].acc.AccZ,\
            "Thigh_Roll_LT":imusignal.signal[3].angle.Roll, "Thigh_Pitch_LT":imusignal.signal[3].angle.Pitch,   "Thigh_Course_LT":imusignal.signal[3].anglerate.Course,\
        },ignore_index=True)
endtime = time.time()
print("cost time:",endtime-starttime)

該方法簡單而且耗時僅長於最快的實現方法，測試耗時爲：0.016989469528198242code

SignalFrame = DataFrame()

for leg in Leg:
    for leg_position in  Leg_Position:
        for signal in  accel + angle:
            SignalFrame[leg_position+"_"+signal+"_"+leg]  = np.arange(999999999,999999999+int(period*rate),1)/1000000
starttime = time.time()
i=0
for imusignal in ImuSignalDataset.imusignal:
    for SignalData in imusignal.signal:
        signal_dict = SignalAttributeDict(SignalData)
        for signal in angle + accel:
            SignalFrame.at[i,LegPositionString(SignalData.legposition)+"_"+signal+"_"+LegString(SignalData.leg)]=signal_dict[signal]
    i+=1
endtime = time.time()
print("cost time:",endtime-starttime)

該方法不簡單，若是數據量較大可能會影響數據的準確度，可是耗時應該是最短的，測試耗時爲：0.012990951538085938內存

SignalFrame = DataFrame()

for leg in Leg:
    for leg_position in  Leg_Position:
        for signal in  accel + angle:
            SignalFrame[leg_position+"_"+signal+"_"+leg]  = np.arange(999999999,999999999+int(period*rate),1)/1000000

starttime = time.time()
i=0
for imusignal in ImuSignalDataset.imusignal:
    SignalFrame.loc[i]=[imusignal.signal[0].acc.AccX   , imusignal.signal[0].acc.AccY    , imusignal.signal[0].acc.AccZ,\
                            imusignal.signal[0].angle.Roll  , imusignal.signal[0].angle.Pitch , imusignal.signal[0].anglerate.Course,\
                            imusignal.signal[1].acc.AccX    , imusignal.signal[1].acc.AccY    , imusignal.signal[1].acc.AccZ,\
                            imusignal.signal[1].angle.Roll  , imusignal.signal[1].angle.Pitch , imusignal.signal[1].anglerate.Course,\
                            imusignal.signal[2].acc.AccX    , imusignal.signal[2].acc.AccY    , imusignal.signal[2].acc.AccZ,\
                            imusignal.signal[2].angle.Roll  , imusignal.signal[2].angle.Pitch , imusignal.signal[2].anglerate.Course,\
                            imusignal.signal[3].acc.AccX    , imusignal.signal[3].acc.AccY    , imusignal.signal[3].acc.AccZ,\
                            imusignal.signal[3].angle.Roll  , imusignal.signal[3].angle.Pitch , imusignal.signal[3].anglerate.Course]
    i+=1
endtime = time.time()
print("cost time:",endtime-starttime)

用於生成ImuEncode_pb2的ImuEncode.proto文件以下數學

syntax = "proto2";
package ImuTutorial;

message Signal {
  enum Leg {
    RT = 0;
    LT = 1;
  }
  enum LegPosition{
    Foot = 0;
    Thigh = 1;
  }

  required Leg leg = 1 [default = RT];
  required LegPosition legposition = 2 [default = Foot];
  optional float TimeStamp = 3;
  optional int64 time = 4;
  optional int32 id = 5;

  message Acc{
    required float AccX = 1;
    required float AccY = 2;
    required float AccZ = 3;
  }
  message Angle{
    optional float Course = 1;
    required float Roll = 2;
    required float Pitch = 3;
  }
  message AngleRate{
    required float Course = 1;
    optional float Roll = 2;
    optional float Pitch = 3;
  }
  message Quaternion{
    optional float v = 1;
    required float x = 2;
    required float y = 3;
    required float z = 4;
  }
  required Acc acc = 6;
  required Angle angle= 7;
  required AngleRate anglerate= 8;
  optional Quaternion quaternion = 9;
}

message ImuSignal {
  required uint64 time = 1;
  repeated Signal signal = 2;  
}

message ImuSignalDataset {
  repeated ImuSignal imusignal = 1;
}

message Datalength {
  required fixed32 length = 1;
}