用memory_profiler 監控內存耗用

時間 2019-11-22

標籤 memory profiler 監控內存耗用简体版

原文原文鏈接

用memory_profiler 監控內存耗用

用於搜索某些文件，好比後綴是.py、.mp3等文件，endswith(".py")便可
root@WJL-SH4031667:/mnt/d/Downloads# cat getspecFile.pypython

python hljs 19行linux

from os import walk
from os.path import join,getmtime

@profile
def scan():
    ret = {}
    for path, _, files in walk("./"):
        for f in files:
            if not f.endswith(""):
                continue

            filename = join(path, f)
            ret[filename] = getmtime(filename)

    return ret

if __name__ == "__main__":
    print(len(scan().keys())個文件)

一. 每行代碼的內存損耗

root@WJL-SH4031667:/mnt/d/Downloads # python -m memory_profiler getspecFile.py
230073個文件
Filename: getspecFile.pyui

python hljs 17行.net

mem-usage列中使用了約96M的內存， increment 增量列中爲何會出現負數，不知道

Line #    Mem usage    Increment   Line Contents
================================================
     4   29.785 MiB   29.785 MiB   @profile
     5                             def scan():
     6   29.789 MiB    0.004 MiB       ret = {}
     7   96.504 MiB -4610.625 MiB       for path, _, files in walk("./"):
     8   96.504 MiB -33928.625 MiB           for f in files:
     9   96.504 MiB -29646.871 MiB               if not f.endswith(""):
    10                                             continue
    11
    12   96.504 MiB -29647.805 MiB               filename = join(path, f)
    13   96.504 MiB -29609.523 MiB               ret[filename] = getmtime(filename)
    14
    15   96.504 MiB    0.000 MiB       return ret

二. 生成內存損耗曲線圖表

1. 首先安裝pylab

root@WJL-SH4031667:/mnt/d/Downloads # pip install matplotlib3d

python hljs 23行code

Looking in indexes: https://pypi.tuna.tsinghua.edu.cn/simple
Collecting matplotlib
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/b2/58/5842588fa67b45ffb451c4c98eda283c0c42b8f2c5e503e4f6d9ff3c3a63/matplotlib-3.0.1-cp35-cp35m-manylinux1_x86_64.whl (12.9MB)
    100% |████████████████████████████████| 12.9MB 821kB/s
Collecting cycler>=0.10 (from matplotlib)
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/f7/d2/e07d3ebb2bd7af696440ce7e754c59dd546ffe1bbe732c8ab68b9c834e61/cycler-0.10.0-py2.py3-none-any.whl
Collecting numpy>=1.10.0 (from matplotlib)
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/98/d6/ebc4f752b683d190361248ecce4d5a0977b5c483370aa7ff63b733e8f011/numpy-1.15.3-cp35-cp35m-manylinux1_x86_64.whl (13.8MB)
    100% |████████████████████████████████| 13.8MB 690kB/s
Collecting pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 (from matplotlib)
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/2b/4a/f06b45ab9690d4c37641ec776f7ad691974f4cf6943a73267475b05cbfca/pyparsing-2.2.2-py2.py3-none-any.whl (57kB)
    100% |████████████████████████████████| 61kB 1.7MB/s
Collecting python-dateutil>=2.1 (from matplotlib)
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/74/68/d87d9b36af36f44254a8d512cbfc48369103a3b9e474be9bdfe536abfc45/python_dateutil-2.7.5-py2.py3-none-any.whl (225kB)
    100% |████████████████████████████████| 235kB 1.6MB/s
Collecting kiwisolver>=1.0.1 (from matplotlib)
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/7e/31/d6fedd4fb2c94755cd101191e581af30e1650ccce7a35bddb7930fed6574/kiwisolver-1.0.1-cp35-cp35m-manylinux1_x86_64.whl (949kB)
    100% |████████████████████████████████| 952kB 2.7MB/s
Requirement already satisfied: six in /usr/lib/python3/dist-packages (from cycler>=0.10->matplotlib) (1.10.0)
Requirement already satisfied: setuptools in /usr/lib/python3/dist-packages (from kiwisolver>=1.0.1->matplotlib) (20.7.0)
Installing collected packages: cycler, numpy, pyparsing, python-dateutil, kiwisolver, matplotlib
Successfully installed cycler-0.10.0 kiwisolver-1.0.1 matplotlib-3.0.1 numpy-1.15.3 pyparsing-2.2.2 python-dateutil-2.7.5

2. 用mprof 來生成一個.dat文件，裏面記錄了內存的損耗

root@WJL-SH4031667:/mnt/d/Downloads # mprof run getspecFile.pyblog

python hljs 4行ip

mprof: Sampling memory every 0.1s
running as a Python program...
230074個文件

3. 查看生成的dat文件

root@WJL-SH4031667:/mnt/d/Downloads # ls *.dat -l
-rwxrwxrwx 1 root root 25035 Oct 31 08:54 mprofile_20181031085302.dat
root@WJL-SH4031667:/mnt/d/Downloads # more mprofile_20181031085302.dat內存

python hljs 31行ci

CMDLINE /usr/bin/python3 getspecFile.py
MEM 0.257812 1540947182.6013
MEM 8.792969 1540947182.7021
MEM 12.125000 1540947182.8037
MEM 15.257812 1540947182.9046
MEM 17.632812 1540947183.0055
MEM 21.906250 1540947183.1060
MEM 24.519531 1540947183.2068
MEM 26.437500 1540947183.3075
MEM 29.242188 1540947183.4081
MEM 29.722656 1540947183.5088
MEM 29.929688 1540947183.6097
MEM 29.984375 1540947183.7102
MEM 29.996094 1540947183.8110
MEM 30.164062 1540947183.9120
MEM 30.179688 1540947184.0127
MEM 30.332031 1540947184.1137
MEM 30.339844 1540947184.2145
MEM 30.339844 1540947184.3152
MEM 30.339844 1540947184.4160
MEM 30.527344 1540947184.5168
MEM 30.527344 1540947184.6176
MEM 30.527344 1540947184.7184
MEM 30.539062 1540947184.8191
MEM 30.562500 1540947184.9199
MEM 30.578125 1540947185.0207
MEM 30.582031 1540947185.1214
MEM 30.605469 1540947185.2225
MEM 30.625000 1540947185.3231
...