在Android中使用FlatBuffers（中篇）

時間 2019-12-13

標籤 android 使用 flatbuffers 中篇欄目 Android 简体版

原文原文鏈接

本文來自網易雲社區。

FlatBuffers、Protobuf及JSON對比測試

FlatBuffers相對於Protobuf的表現又如何呢？這裏咱們用數聽說話，對比一下FlatBuffers格式、JSON格式與Protobuf的表現。測試一樣用fastjson做爲JSON的編碼解碼工具。java

測試用的數據結構全部的數據結構，Protobuf相關的測試代碼，及JSON的測試代碼同在Android中使用Protocol Buffers 一文所述，FlatBuffers的測試代碼如上面看到的 AddressBookFlatBuffers。json

經過以下的這段代碼來執行測試：數組

private class ProtoTestTask extends AsyncTask<Void, Void, Void> {
        private static final int BUFFER_LEN = 8192;

        private void compress(InputStream is, OutputStream os)
                throws Exception {

            GZIPOutputStream gos = new GZIPOutputStream(os);

            int count;
            byte data[] = new byte[BUFFER_LEN];
            while ((count = is.read(data, 0, BUFFER_LEN)) != -1) {
                gos.write(data, 0, count);
            }

            gos.finish();
            gos.close();
        }

        private int getCompressedDataLength(byte[] data) {
            ByteArrayInputStream bais =new ByteArrayInputStream(data);
            ByteArrayOutputStream baos = new ByteArrayOutputStream();try {
                compress(bais, baos);
            } catch (Exception e) {
            }

            return baos.toByteArray().length;
        }

        private void dumpDataLengthInfo(byte[] protobufData, String jsonData, byte[] flatbufData) {
            int compressedProtobufLength = getCompressedDataLength(protobufData);
            int compressedJSONLength = getCompressedDataLength(jsonData.getBytes());
            int compressedFlatbufLength = getCompressedDataLength(flatbufData);
            Log.i(TAG, String.format("%-120s", "Data length"));
            Log.i(TAG, String.format("%-20s%-20s%-20s%-20s%-20s%-20s", "Protobuf", "Protobuf (GZIP)",
                    "JSON", "JSON (GZIP)", "Flatbuf", "Flatbuf (GZIP)"));
            Log.i(TAG, String.format("%-20s%-20s%-20s%-20s%-20s%-20s",
                    String.valueOf(protobufData.length), compressedProtobufLength,
                    String.valueOf(jsonData.getBytes().length), compressedJSONLength,
                    String.valueOf(flatbufData.length), compressedFlatbufLength));
        }

        private void doEncodeTest(String[] names, int times) {
            long startTime = System.nanoTime();
            byte[] protobufData = AddressBookProtobuf.encodeTest(names, times);
            long protobufTime = System.nanoTime();
            protobufTime = protobufTime - startTime;

            startTime = System.nanoTime();
            String jsonData = AddressBookJson.encodeTest(names, times);
            long jsonTime = System.nanoTime();
            jsonTime = jsonTime - startTime;

            startTime = System.nanoTime();
            byte[] flatbufData = AddressBookFlatBuffers.encodeTest(names, times);
            long flatbufTime = System.nanoTime();
            flatbufTime = flatbufTime - startTime;

            dumpDataLengthInfo(protobufData, jsonData, flatbufData);

            Log.i(TAG, String.format("%-20s%-20s%-20s%-20s", "Encode Times", String.valueOf(times),
                    "Names Length", String.valueOf(names.length)));

            Log.i(TAG, String.format("%-20s%-20s%-20s%-20s%-20s%-20s",
                    "ProtobufTime", String.valueOf(protobufTime),
                    "JsonTime", String.valueOf(jsonTime),
                    "FlatbufTime", String.valueOf(flatbufTime)));
        }

        private void doEncodeTest10(int times) {
            doEncodeTest(TestUtils.sTestNames10, times);
        }

        private void doEncodeTest50(int times) {
            doEncodeTest(TestUtils.sTestNames50, times);
        }

        private void doEncodeTest100(int times) {
            doEncodeTest(TestUtils.sTestNames100, times);
        }

        private void doEncodeTest(int times) {
            doEncodeTest10(times);
            doEncodeTest50(times);
            doEncodeTest100(times);
        }

        private void doDecodeTest(String[] names, int times) {
            byte[] protobufBytes = AddressBookProtobuf.encodeTest(names);
            ByteArrayInputStream bais = new ByteArrayInputStream(protobufBytes);
            long startTime = System.nanoTime();
            AddressBookProtobuf.decodeTest(bais, times);
            long protobufTime = System.nanoTime();
            protobufTime = protobufTime - startTime;

            String jsonStr = AddressBookJson.encodeTest(names);
            startTime = System.nanoTime();
            AddressBookJson.decodeTest(jsonStr, times);
            long jsonTime = System.nanoTime();
            jsonTime = jsonTime - startTime;

            byte[] flatbufData = AddressBookFlatBuffers.encodeTest(names);
            startTime = System.nanoTime();
            AddressBookFlatBuffers.decodeTest(flatbufData, times);
            long flatbufTime = System.nanoTime();
            flatbufTime = flatbufTime - startTime;

            Log.i(TAG, String.format("%-20s%-20s%-20s%-20s", "Decode Times", String.valueOf(times),
                    "Names Length", String.valueOf(names.length)));
            Log.i(TAG, String.format("%-20s%-20s%-20s%-20s%-20s%-20s",
                    "ProtobufTime", String.valueOf(protobufTime),
                    "JsonTime", String.valueOf(jsonTime),
                    "FlatbufTime", String.valueOf(flatbufTime)));
        }

        private void doDecodeTest10(int times) {
            doDecodeTest(TestUtils.sTestNames10, times);
        }

        private void doDecodeTest50(int times) {
            doDecodeTest(TestUtils.sTestNames50, times);
        }

        private void doDecodeTest100(int times) {
            doDecodeTest(TestUtils.sTestNames100, times);
        }

        private void doDecodeTest(int times) {
            doDecodeTest10(times);
            doDecodeTest50(times);
            doDecodeTest100(times);
        }

        @Override
        protected Void doInBackground(Void... params) {
            TestUtils.initTest();
            doEncodeTest(5000);

            doDecodeTest(5000);
            return null;
        }

        @Override
        protected void onPostExecute(Void aVoid) {
            super.onPostExecute(aVoid);
        }
    }

這裏咱們執行3組編碼測試及3組解碼測試。對於編碼測試，第一組的單個數據中包含10個Person，第二組的包含50個，第三組的包含100個，而後對每一個數據分別執行5000次的編碼操做。數據結構

對於解碼測試，三組中單個數據一樣包含10個Person、50個及100個，而後對每一個數據分別執行5000次的解碼碼操做。ide

在Galaxy Nexus的Android 4.4.4 CM平臺上執行上述測試，最終獲得以下結果：工具

編碼後數據長度對比 (Bytes)

Person個數	Protobuf	Protobuf（GZIP）	JSON	JSON（GZIP）	Flatbuf	Flatbuf（GZIP)
10	860	288	1703	343	1532	513
50	4300	986	8463	1048	7452	1814
100	8600	1841	16913	1918	14852	3416

相同的數據，通過編碼，在壓縮前JSON的數據最長，FlatBuffers的數據長度與JSON的短大概10 ％，而Protobuf的數據長度則大概只有JSON的一半。而在用GZIP壓縮後，Protobuf的數據長度與JSON的接近，FlatBuffers的數據長度則接近二者的兩倍。性能

編碼性能對比 (S)

Person個數	Protobuf	JSON	FlatBuffers
10	6.000	8.952	12.464
50	26.847	45.782	56.752
100	50.602	73.688	108.426

編碼性能Protobuf相對於JSON有較大幅度的提升，而FlatBuffers則有較大幅度的下降。測試

解碼性能對比 (S)

Person個數	Protobuf	JSON	FlatBuffers
10	0.255	10.766	0.014
50	0.245	51.134	0.014
100	0.323	101.070	0.006

解碼性能方面，Protobuf相對於JSON，有着驚人的提高。Protobuf的解碼時間幾乎不隨着數據長度的增加而有太大的增加，而JSON則隨着數據長度的增長，解碼所須要的時間也愈來愈長。而FlatBuffers則因爲無需解碼，在性能方面相對於前二者更有着很是大的提高。ui

FlatBuffers 編碼原理

FlatBuffers的Java庫只提供了以下的4個類：this

./com/google/flatbuffers/Constants.java
./com/google/flatbuffers/FlatBufferBuilder.java
./com/google/flatbuffers/Struct.java
./com/google/flatbuffers/Table.java

Constants 類定義FlatBuffers中可用的基本原始數據類型的長度：

public class Constants {
    // Java doesn't seem to have these.
    /** The number of bytes in an `byte`. */
    static final int SIZEOF_BYTE = 1;
    /** The number of bytes in a `short`. */
    static final int SIZEOF_SHORT = 2;
    /** The number of bytes in an `int`. */
    static final int SIZEOF_INT = 4;
    /** The number of bytes in an `float`. */
    static final int SIZEOF_FLOAT = 4;
    /** The number of bytes in an `long`. */
    static final int SIZEOF_LONG = 8;
    /** The number of bytes in an `double`. */
    static final int SIZEOF_DOUBLE = 8;
    /** The number of bytes in a file identifier. */
    static final int FILE_IDENTIFIER_LENGTH = 4;
}

FlatBufferBuilder 用於FlatBuffers編碼，它會將咱們的結構化數據序列化爲字節數組。咱們藉助於 FlatBufferBuilder 在 ByteBuffer 中放置基本數據類型的數據、數組、字符串及對象。ByteBuffer 用於處理字節序，在序列化時，它將數據按適當的字節序進行序列化，在發序列化時，它將多個字節轉換爲適當的數據類型。在 .fbs 文件中定義的 table 和 struct，爲它們生成的Java 類會繼承 Table 和 Struct。

在反序列化時，輸入的ByteBuffer數據被看成字節數組，Table提供了針對字節數組的操做，生成的Java類負責對這些數據進行解釋。對於FlatBuffers編碼的數據，無需進行解碼，只需進行解釋。在編譯 .fbs 文件時，每一個字段在這段數據中的位置將被肯定。每一個字段的類型及長度將被硬編碼進生成的Java類。

Struct 類的代碼也比較簡潔：

package com.google.flatbuffers;

import java.nio.ByteBuffer;

/// @cond FLATBUFFERS_INTERNAL

/**
 * All structs in the generated code derive from this class, and add their own accessors.
 */
public class Struct {
  /** Used to hold the position of the `bb` buffer. */
  protected int bb_pos;
  /** The underlying ByteBuffer to hold the data of the Struct. */
  protected ByteBuffer bb;
}

總體的結構以下圖：

在序列化結構化數據時，咱們首先須要建立一個 FlatBufferBuilder ，在這個對象的建立過程當中會分配或從調用者那裏獲取 ByteBuffer，序列化的數據將保存在這個 ByteBuffer中：

/**
    * Start with a buffer of size `initial_size`, then grow as required.
    *
    * @param initial_size The initial size of the internal buffer to use.
    */
    public FlatBufferBuilder(int initial_size) {
        if (initial_size <= 0) initial_size = 1;
        space = initial_size;
        bb = newByteBuffer(initial_size);
    }

   /**
    * Start with a buffer of 1KiB, then grow as required.
    */
    public FlatBufferBuilder() {
        this(1024);
    }

    /**
     * Alternative constructor allowing reuse of {@link ByteBuffer}s.  The builder
     * can still grow the buffer as necessary.  User classes should make sure
     * to call {@link #dataBuffer()} to obtain the resulting encoded message.
     *
     * @param existing_bb The byte buffer to reuse.
     */
    public FlatBufferBuilder(ByteBuffer existing_bb) {
        init(existing_bb);
    }

    /**
     * Alternative initializer that allows reusing this object on an existing
     * `ByteBuffer`. This method resets the builder's internal state, but keeps
     * objects that have been allocated for temporary storage.
     *
     * @param existing_bb The byte buffer to reuse.
     * @return Returns `this`.
     */
    public FlatBufferBuilder init(ByteBuffer existing_bb){
        bb = existing_bb;
        bb.clear();
        bb.order(ByteOrder.LITTLE_ENDIAN);
        minalign = 1;
        space = bb.capacity();
        vtable_in_use = 0;
        nested = false;
        finished = false;
        object_start = 0;
        num_vtables = 0;
        vector_num_elems = 0;
        return this;
    }

    static ByteBuffer newByteBuffer(int capacity) {
        ByteBuffer newbb = ByteBuffer.allocate(capacity);
        newbb.order(ByteOrder.LITTLE_ENDIAN);
        return newbb;
    }

下面咱們更詳細地分析基本數據類型數據、數組及對象的序列化過程。ByteBuffer 爲小尾端的。

FlatBuffers編碼基本數據類型

FlatBuffer 的基本數據類型主要包括以下這些：

Boolean
Byte
Short
Int
Long
Float
Double

FlatBufferBuilder 提供了三組方法用於操做這些數據：

public void putBoolean(boolean x);
    public void putByte   (byte    x);
    public void putShort  (short   x);
    public void putInt    (int     x);
    public void putLong   (long    x);
    public void putFloat  (float   x);
    public void putDouble (double  x);

    public void addBoolean(boolean x);
    public void addByte   (byte    x);
    public void addShort  (short   x);
    public void addInt    (int     x);
    public void addLong   (long    x);
    public void addFloat  (float   x);
    public void addDouble (double  x);

    public void addBoolean(int o, boolean x, boolean d);
    public void addByte(int o, byte x, int d);
    public void addShort(int o, short x, int d);
    public void addInt    (int o, int     x, int     d);
    public void addLong   (int o, long    x, long    d);
    public void addFloat  (int o, float   x, double  d);
    public void addDouble (int o, double  x, double  d);

putXXX 那一組，直接地將一個數據放入 ByteBuffer 中，它們的實現基本以下面這樣：

public void putBoolean(boolean x) {
        bb.put(space -= Constants.SIZEOF_BYTE, (byte) (x ? 1 : 0));
    }

    public void putByte(byte x) {
        bb.put(space -= Constants.SIZEOF_BYTE, x);
    }

    public void putShort(short x) {
        bb.putShort(space -= Constants.SIZEOF_SHORT, x);
    }

Boolean值會被先轉爲byte類型再放入 ByteBuffer。另一點值得注意的是，數據是從 ByteBuffer 的結尾處開始放置的，space用於記錄最近放入的數據的位置及剩餘的空間。

addXXX(XXX x) 那一組在放入數據以前會先作對齊處理，並在須要時擴展 ByteBuffer 的容量：

static ByteBuffer growByteBuffer(ByteBuffer bb) {
        int old_buf_size = bb.capacity();
        if ((old_buf_size & 0xC0000000) != 0)  // Ensure we don't grow beyond what fits in an int.
            throw new AssertionError("FlatBuffers: cannot grow buffer beyond 2 gigabytes.");
        int new_buf_size = old_buf_size << 1;
        bb.position(0);
        ByteBuffer nbb = newByteBuffer(new_buf_size);
        nbb.position(new_buf_size - old_buf_size);
        nbb.put(bb);
        return nbb;
    }

   public void pad(int byte_size) {
       for (int i = 0; i < byte_size; i++) bb.put(--space, (byte) 0);
   }

    public void prep(int size, int additional_bytes) {
        // Track the biggest thing we've ever aligned to.
        if (size > minalign) minalign = size;
        // Find the amount of alignment needed such that `size` is properly
        // aligned after `additional_bytes`
        int align_size = ((~(bb.capacity() - space + additional_bytes)) + 1) & (size - 1);
        // Reallocate the buffer if needed.
        while (space < align_size + size + additional_bytes) {
            int old_buf_size = bb.capacity();
            bb = growByteBuffer(bb);
            space += bb.capacity() - old_buf_size;
        }
        pad(align_size);
    }

    public void addBoolean(boolean x) {
        prep(Constants.SIZEOF_BYTE, 0);
        putBoolean(x);
    }

    public void addInt(int x) {
        prep(Constants.SIZEOF_INT, 0);
        putInt(x);
    }

對齊是數據存放的起始位置相對於ByteBuffer的結束位置的對齊，additional bytes被認爲是不須要對齊的，且在必要的時候會在ByteBuffer可用空間的結尾處填充值爲0的字節。在擴展 ByteBuffer 的空間時，老的ByteBuffer被放在新ByteBuffer的結尾處。

addXXX(int o, XXX x, YYY y) 這一組方法在放入數據以後，會將 vtable 中對應位置的值更新爲最近放入的數據的offset。

public void addShort(int o, short x, int d) {
        if (force_defaults || x != d) {
            addShort(x);
            slot(o);
        }
    }

    public void slot(int voffset) {
        vtable[voffset] = offset();
    }

後面咱們在分析編碼對象時再來詳細地瞭解vtable。

基本上，在咱們的應用程序代碼中不要直接調用這些方法，它們主要在構造對象時用於存儲對象的基本數據類型字段。