速記符	匹配項
ip	dl_type=0x800
ipv6	dl_type=0x86dd
icmp	dl_type=0x0800,nw_proto=1
icmp6	dl_type=0x86dd,nw_proto=58
tcp	dl_type=0x0800,nw_proto=6
tcp6	dl_type=0x86dd,nw_proto=6
udp	dl_type=0x0800,nw_proto=17
udp6	dl_type=0x86dd,nw_proto=17
sctp	dl_type=0x0800,nw_proto=132
sctp6	dl_type=0x86dd,nw_proto=132
arp	dl_type=0x0806
rarp	dl_type=0x8035
mpls	dl_type=0x8847
mplsm	dl_type=0x8848

NXM字段	報文字段
NXM_OF_ETH_SRC	源MAC
NXM_OF_ETH_DST	目的MAC
NXM_OF_ETH_TYPE	以太網類型
NXM_OF_VLAN_TCI	vid
NXM_OF_IP_PROTO	IP協議號
NXM_OF_IP_TOS	IP ToS值
NXM_NX_IP_ECN	IP ToS ECN
NXM_OF_IP_SRC	源IP
NXM_OF_IP_DST	目的IP
NXM_OF_TCP_SRC	TCP源端口
NXM_OF_TCP_DST	TCP目的端口
NXM_OF_UDP_SRC	UDP源端口
NXM_OF_UDP_DST	UDP目的端口
NXM_OF_SCTP_SRC	SCTP源端口
NXM_OF_SCTP_DST	SCTP目的端口

2.設計考慮

2.1.總體數據結構

ovs datapath classifier涉及的數據結構主要有以下。

網橋數據結構

struct datapath {

struct rcu_head rcu;

/* 該結構使用雙鏈表組織 */

struct list_head list_node;

/* 從屬該datapath的流表項 */

struct flow_table table;

/* 從屬該datapath的vport信息 */

struct hlist_head *ports;

…

}

流表數據結構

struct flow_table {

/* 用key組織的流表實例 */

struct table_instance *ti;

/* 用unique flow identifier(ufid)組織的流表實例 */

struct table_instance *ufid_ti;

/* 掩碼信息緩存表 */

struct mask_cache_entry *mask_cache;

/* 掩碼信息列表 */

struct mask_array *mask_array;

/* 記錄當前節拍數 */

unsigned long last_rehash;

/* ti流表實例中存儲的流表項數目 */

unsigned int count;

/* ufid_ti流表實例中存儲的流表項數目 */

unsigned int ufid_count;

}

流表實例數據結構

struct table_instance {

/* 哈希桶,用於組織各個流表項的具體信息. */

struct flex_array *buckets;

/* 哈希桶大小 */

unsigned int n_buckets;

struct rcu_head rcu;

int node_ver;

u32 hash_seed;

bool keep_flows;

}

掩碼信息列表數據結構

struct mask_array {

struct rcu_head rcu;

/* 當前已存儲的掩碼信息數目 */

int count;

/* 一共能夠存儲掩碼信息的總數目 */

int max;

/* 具體的掩碼信息 */

struct sw_flow_mask *masks[];

}

struct sw_flow_mask {

/* 引用計數 */

int ref_count;

struct rcu_head rcu;

/* 匹配關鍵字的有效範圍,下文會具體解釋 */

struct sw_flow_key_range range;

/* 匹配關鍵字信息 */

struct sw_flow_key key;

}

掩碼信息緩存表數據結構

struct mask_cache_entry {

/* hash值 */

u32 skb_hash;

/* 對應哪個掩碼信息,索引值 */

u32 mask_index;

}

哈希桶數據結構

struct flex_array {

union {

struct {

/* 實際等於sizeof(struct hlist_head) */

int element_size;

/* 一共有多少個struct hlist_head指針. */

int total_nr_elements;

/* 每一個part(頁)能夠存儲多少個struct hlist_head指針 */

int elems_per_part;

u32 reciprocal_value reciprocal_elems;

/* 具體存儲struct hlist_head指針的緩衝區 */

struct flex_array_part *parts[];

};

/* 填充字段.表明了整個結構的大小=頁大小(4096字節) */

char padding[FLEX_ARRAY_BASE_SIZE];

}

struct flex_array_part {

/* 緩衝區.一個頁大小,即:4096字節 */

char elements[FLEX_ARRAY_PART_SIZE];

};

上述這些數據結構是在源碼中的ovs_flow_tbl_init函數裏面進行初始化操做的。初始化後以及上述這些數據結構之間的關係以下圖所示。

2.2.關鍵信息範圍

從上面2.1節可知，掩碼信息(struct sw_flow_mask結構)中記錄了一個範圍(struct sw_flow_key_range結構)，該範圍用於標識須要匹配的關鍵信息最小偏移和最大偏移。爲何須要這樣作？我的感受和ovs-dpdk datapath classifier中描述的miniflow相似，即：匹配過程當中並不是使用報文的全部字段去匹配，而是使用部分字段，例如使用報文的五元組信息(源IP、目的IP、協議號、源端口、目的端口)。那麼使用sw_flow_key_range結構來標識這五元組信息中最小偏移和最大偏移。實際源碼中，關鍵信息是使用struct sw_flow_key結構來描述的。因爲該結構字段較多，這裏不詳細給出。以報文五元組信息爲例，這裏給出的五元組信息所在sw_flow_key結構的位置和實際源碼對應的位置是不相同的，這裏只是給出計算最小偏移和最大偏移的概念，以下圖所示，關鍵信息的有效範圍爲：
最小偏移=M，最大偏移=N

2.3.更新過程

在源碼中，對應更新過程的入口函數是：ovs_flow_cmd_new。這個入口函數是處於內核模塊中，在接收到報文時，經過下面2.4節所述的查找過程，查找失敗時，會將報文的相關信息upcall到用戶空間，在用戶空間經過查找」慢路徑」將對應的actions和mask信息下發到內核空間，在內核空間，處理的入口函數正是ovs_flow_cmd_new。下面將對這個函數的處理過程做詳細描述。描述以前，先掌握一些相關的數據結構，以下所示：

struct sw_flow {

struct rcu_head rcu;

/* 哈希桶鏈表 */

struct {

struct hlist_node node[2];

u32 hash;

} flow_table, ufid_table;

int stats_last_writer;

/* 關鍵信息.通過掩碼運算後的 */

struct sw_flow_key key;

/* 流id信息 */

struct sw_flow_id id;

/* 指向的掩碼信息 */

struct sw_flow_mask *mask;

/* 指向的流動做信息 */

struct sw_flow_actions *sf_acts;

struct flow_stats *stats[];

}

struct sw_flow_id {

u32 ufid_len;

union {

/* ufid爲128比特,共16字節 */

u32 ufid[4];

struct sw_flow_key *unmasked_key;

}

struct sw_flow_actions {

struct rcu_head rcu;

size_t orig_len;

u32 actions_len;

struct nlattr actions[];

}

struct sw_flow_match {

struct sw_flow_key *key;

struct sw_flow_key_range range;

struct sw_flow_mask *mask;

}

具體流程以下所示：
1)初始化struct sw_flow *new_flow。分配空間給new_flow，並設置部分的成員值，以下：
new_flow->sf_acts = null;
new_flow->mask = null;
new_flow->id.ufid_len=0;
new_flow->id.unmasked_key=null;
2)構建struct sw_flow_match match信息。根據接收到的關鍵信息和掩碼信息，存儲在match.key和match.mask中。

3)計算new_flow->key = match.key & match.mask。
4)將ufid信息提取出來存儲在new_flow->id.ufid_len和new_flow->id.ufid中。若是未提供任何的UFID信息(此時的new_flow->id.ufid_len爲0)，則將match.key信息拷貝一份到new_flow->id.unmasked_key中存儲。
5)將actions信息存儲在變量struct sw_flow_actions acts中。
6)根據接收到的dp_ifindex獲取struct datapath dp信息。
7)若是new_flow->id.ufid_len不爲0，則使用new_flow->id.ufid信息去dp->table.ufid_ti指向的哈希桶中找到對應的struct sw_flow信息。假設找不到。
8)而若是new_flow->id.ufid_len爲0，則須要在dp->table.ti，同時配合在2)中構建出來的match.key和dp->table.mask_array配合找出對應的struct sw_flow信息。這個詳細過程能夠參考下面的2.4節。這裏也假設找不到。
9)通過上面7)和8)步都沒找到對應的struct sw_flow信息。首先設置new_flow->sf_acts=acts(在5)步中獲取)。而後使用match.mask去dp->table.mask_array中查找是否已存在該match.mask信息。若是不存在則接着分配一個struct sw_flow_mask *mask。設置mask->key=match.mask->key, mask->range=match.mask->range, mask->ref_count=1。而且在dp->table.mask_array.masks[i]中存儲該mask的地址。添加完畢以後，以下圖所示。變化的地方用綠色標註出來了。同時new_flow->mask=mask。

10)根據new_flow->key和new_flow->mask.range計算出new_flow->flow_table.hash值。
11)在dp->table.ti->buckets指向的哈希桶中插入new_flow。以下圖所示(假設插入的位置爲part[0]對應的第0個位置)。dp->table.count。
12)若是new_flow->id.ufid_len不爲0，則使用new_flow->id.ufid信息計算出new_flow->ufid_table.hash。而後根據這個hash值在dp->table.ufid_ti->buckets中找到合適的哈希桶存儲對應的new_flow信息。這裏假設找到的位置爲parts[0]的第0個位置。以下圖所示。插入new_flow信息以後，dp->table.ufid_count。

到此爲止，更新過程完畢。

2.4.查找過程

在源碼中，對應查找過程的入口函數是：ovs_vport_receive。具體過程以下：
1)從接收到的報文信息中提取出關鍵信息。存儲到struct sw_flow_key key。
2)從接收到的報文信息中獲取struct vport信息，再從vport信息獲取struct datapath信息，假設爲dp。
3)若是接收到的報文中未帶有skb_hash值信息，則執行：
遍歷每個掩碼，即:dp->table.mask_array->masks[i]。這裏簡單記爲mask。將1)提取出來的關鍵新key與這個掩碼mask作邏輯與運算，得出掩碼過的信息，記爲masked_key。經過masked_key和mask.range信息計算出hash值在dp->table.ti->buckets中找到對應的哈希桶，並遍歷該桶上的全部struct sw_flow信息，記爲flow。執行比較：若是flow->mask指向的是mask，flow->flow_table.hash值和計算出的hash值相等，且flow->key和masked_key信息在mask.range指定的範圍內徹底相同，則認爲成功匹配，返回flow信息。不然匹配失敗，返回NULL。
4)若是接收到的報文中帶有skb_hash值信息，則執行：
4)-1：根據舊的skb_hash值和key->recirc_id值從新計算出新的skb_hash值。
4)-2：將skb_hash值分紅4段，每段1字節，記爲hash。遍歷每一段的hash值，執行：獲取dp->table.mask_cache[hash]表項，記爲entry，若是entry.skb_hash值和skb_hash值相等，則使用entry.mask_index指向的掩碼信息去dp->table.ti->buckets中找到對應的struct sw_flow信息(過程和上述第3)步相同)。若是找到對應的flow，則返回flow，過程當中會更新entry.mask_index值來實際指向具體的掩碼信息索引。不然，匹配失敗，將entry.skb_hash值置0，返回NULL。而若是每個entry.skb_hash值和skb_hash值不相等，則遍歷完每一段hash值以後，從中選擇最佳候選entry，最佳候選的條件是：其skb_hash值最小者，假設記爲ce。最後使用ce.mask_index指向的掩碼信息去dp->table.ti->buckets中找到對應的struct sw_flow信息(過程和上述第3)步相同)。若是找到對應的flow，則返回flow，同時將ce->skb_hash值更新爲skb_hash值。固然在查找的過程當中(上述第3)步)，也會更新ce->mask_index來指向實際的掩碼信息索引。
5) 若是在上述第3)和第4)步中都匹配失敗了，則須要將報文的信息upcall到用戶空間，由用戶空間負責查找對應的流動做信息，返回給內核。這個過程具體請參考上述2.3節。
到此爲止，查找過程結束。

2.5.執行過程

在2.4節查找過程當中，若是查找成功，則須要執行對應的流動做。入口函數是：ovs_execute_actions。
而若是查找失敗，upcall到用戶空間，找到對應的流動做以後，調用執行，到內核空間，入口函數對應的是：ovs_packet_cmd_execute。該函數最終會調用到ovs_execute_actions。

2.6.掩碼信息比較

在上述2.3和2.4節所述的更新過程和查找過程當中都會碰見用新構造的掩碼信息與datapath結構的table.mask_array做比較，以檢測掩碼信息是否須要新增。而比較的方法以下：
假設新構造的掩碼信息爲mask，與table.mask_array->masksi比較：
1)mask.range.start和exist_mask.range.start相等。
2)mask.range.end和exist_mask.range.end相等。
3)mask和exist_mask在range範圍內徹底相同。
同時符合上面三個條件才認爲兩個掩碼徹底相同。

2.7.流信息比較

在上述2.4節所述的查找過程當中，對接收到的報文進行匹配表項時，須要對流信息做比較。假設接收到的報文提取出來的關鍵信息爲key，匹配的掩碼信息爲mask，經過mask & key計算出掩碼後的報文關鍵信息，記爲masked_key，經過masked_key和mask.range計算出hash值。根據這個hash值找到對應的哈希桶，遍歷這個哈希桶中存儲的每個流信息，記爲flow。如今須要比較flow信息進而找到匹配的流表項。比較以下：
1)flow->mask指向的是mask。
2)flow->flow_table.hash和計算出的hash值相等。
3)flow->key和masked_key在mask.range範圍內徹底相同。
同時符合上面三個條件才認爲找到匹配的流表項。

2.8.mask_cache表項

在ovs_dp_process_packet函數中查找匹配的流表項時，若是報文的skb中已經帶有skb_hash值，則將這個skb_hash值(32比特)分爲4段，每段8比特，每段的哈希值暫記爲hash，用這個hash值去datapath結構中的table.mask_cache緩存表中查找對應的掩碼信息索引。每一個表項(struct mask_cache_entry結構)存儲了skb_hash和mask_index信息。初始化的時候，這個緩存表中全部表項都置0，所以，用報文的skb_hash值，分4段去查找，都沒法找到合適的表項。這時須要從中選出最佳候選的表項，而最佳候選的表項爲其skb_hash值最小。所以，初始化的時候，最終會選擇table.mask_cache[0]爲最佳候選表項。
接着根據上述2.4節所述的查找過程，找到合適的流表項信息。若是匹配成功，則最佳候選表項table.mask_cache[0].mask_index記錄了掩碼信息索引(index)，即：table.mask_array->masks[index]。同時table.mask_cache[0].skb_hash賦值爲skb_hash。假設index爲0，則對應以下圖所示。
後續若是收到的報文帶的skb_hash值與table.mask_cache[0].skb_hash值相等時，則首先使用table.mask_cache[0].mask_index索引的掩碼信息去找匹配的流表信息。固然，若是匹配成功，table.mask_cache[0].mask_index可能並不是爲0(以前存儲的值)，有可能更新爲其餘值。而若是匹配失敗，則table.mask_cache[0].skb_hash置0。

2.9.mask_array擴充

當datapath結構中的table.mask_array->count >= table.mask_array->max時，則須要擴充mask_array空間。已當前table.mask_array->max * 2的大小進行擴充。擴充先後的效果以下圖所示。橙色線指向的掩碼信息在擴充以後，會釋放掉old的掩碼信息空間。擴充老是按照當前max數值的2倍大小進行擴充，例如：16 -> 32 -> 64 -> 128 -> …。從源碼中暫時未看到這個擴充的最大值。

2.10.table_instance擴充

table_instance擴充有兩個條件觸發：
1)當datapath結構中的table.count > table.ti->n_buckets時，觸發擴充。
2)datapath結構中的table.last_rehash記錄了上次執行擴充或初始化table時系統的jiffies值。若是超過10分鐘，則須要從新擴充，只是此次的擴充並不是增大空間，而是以相同的大小從新分配空間。以爲這樣作的意義是從新生成table_instance結構中的hash_seed值，從新安排哈希桶的鏈表長度，分散存儲，減小匹配的比較次數。
咱們主要以第一種條件爲例，描述擴充的過程。這種擴充是在原有的哈希桶數目(n_buckets)基礎之上，以2倍的大小進行擴充。擴充先後，table_instance結構的變化以下圖所示。

根據上圖可知，擴充以後，table_instance中的hash_seed更新爲新的隨機數了。這樣，在從舊的table_instance將哈希桶中對應的各個流信息拷貝到新的table_instance時，須要從新計算哈希桶的位置，從新安排了。這樣作的好處時：能夠從新分散每一個哈希桶中流信息鏈表的長度，減小在匹配時流的比較次數。以下圖所示，以前Flow_A和Flow_B都位於第0個哈希桶，擴充以後，Flow_A處於了第0個哈希桶，而Flow_B則處於第1025個哈希桶了。這樣在查找Flow_B的時候，比較的次數就減小了一次。在大規模查找的過程當中，這種改變能夠大大提升查找的效率。

2.11.示例

假設ovs用戶空間的」慢路徑」存儲的流表信息以下所示：

table=0, src_ip=11.11.11.0/24, dst_ip=192.0.0.0/8, actions=output:1

table=0, src_ip=2.2.2.0/24, dst_ip=2.0.0.0/8, actions=output:2

table=0, dst_ip=8.0.0.0/8, actions=drop

初始化的時候，ovs內核空間的」快路徑」沒有存儲任何的流路徑信息。以下圖所示：

收到第一個報文：src_ip=11.11.11.25, dst_ip=192.1.1.1
收到第一個報文，匹配結果Miss，upcall到用戶空間，查表將結果發送回內核空間的datapath。根據上述2.3節所述的更新過程，會新增相應的掩碼信息(mask_A)和流信息(flow_A)。以下圖所示。

收到第二個報文：src_ip=11.11.11.63, dst_ip=192.168.7.8
根據上述2.4節所述的查找過程，匹配成功。

收到第三個報文：src_ip=2.2.2.4, dst_ip=2.7.7.7
根據上述2.4節所述的查找過程，匹配失敗。由於掩碼信息和mask_A徹底相同，所以無需新增掩碼信息，只須要將mask_A的ref_count引用計數加1便可。可是須要新增相應的流信息(flow_B)，以下圖所示。

收到第四個報文：dst_ip=8.12.34.56
根據上述2.4節所述的查找過程，匹配失敗。須要新增相應的掩碼信息(mask_B)和流信息(flow_C)。以下圖所示

As part of my work in OpenDaylight, we are looking at creating a router using Open vSwitch... Why? Well OpenStack requires some limited L3 capabilities and we think that we can handle those in a distributed router.

Test Topology

My test topology looks like this:

We have a host in an external network 172.16.1.0/24, one host in an internal network 10.10.10.0/24 and two hosts in another internal network 10.10.20.0/24.

As such, The hosts in the 10.x.x.x range should be able to speak to each other, but should not be able to speak to external hosts.

The host 10.10.10.2 has a floating IP of 172.16.1.10 and should be reachable on this address from the external 172.16.1.0/24network. To do this, we'll use DNAT for traffic from 172.16.1.2 -> 172.16.1.10 and SNAT for traffic back from10.10.10.2 -> 172.16.1.2

If you'd like to recreate this topology you can checkout the OpenDaylight OVSDB project source on GitHub and:

 
               vagrant up mininet
vagrant ssh mininet
cd /vagrant/resources/mininet
sudo mn --custom topo.py --topo l3 
              

The Pipeline

Our router is implemented using the following pipeline:

Table 0 - Classifier

In this table we work out what traffic is interesting for us before pushing it further along the pipeline

Table 100 - ACL

While we don't use this table today, the idea would be to filter traffic in this table (or series of tables) and to then resubmit to the classifier once we have scrubbed them.

Table 105 - ARP Responder

In this table we use some OVS-Jitsu to take an incoming ARP Request and turn it in to an ARP reply

Table 5 - L3 Rewrite

In this table, we make any L3 modifications we need to before a packet is routed

Table 10 - L3 Routing

The L3 routing table is where the routing magic happens. Here we modify the Source MAC address, Decrement the TTL and push to the L2 tables for forwarding

Table 15 - L3 Forwarding

In the L3 forwarding table we resolve a destination IP address to the correct MAC address for L2 forwarding.

Table 20 - L2 Rewrites

While we aren't using this table in this example, typically we would push/pop any L2 encapsulations here like a VLAN tag or a VXLAN/GRE Tunnel ID.

Table 25 - L2 Forwarding

In this table we do our L2 lookup and forward out the correct port. We also handle L2 BUM traffic here using OpenFlow Groups - no VLANs!

The Flows

To program the flows, paste the following in to mininet:

## Set Bridge to use OpenFlow 1.3
sh ovs-vsctl set Bridge s1 "protocols=OpenFlow13"

## Create Groups
sh ovs-ofctl add-group -OOpenFlow13 s1 group_id=1,type=all,bucket=output:1
sh ovs-ofctl add-group -OOpenFlow13 s1 group_id=2,type=all,bucket=output:2,4
sh ovs-ofctl add-group -OOpenFlow13 s1 group_id=3,type=all,bucket=output:3

## Table 0 - Classifier
# Send ARP to ARP Responder
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=1000, dl_type=0x0806, actions=goto_table=105"
# Send L3 traffic to L3 Rewrite Table
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=100, dl_dst=00:00:5E:00:02:01, action=goto_table=5"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=100, dl_dst=00:00:5E:00:02:02, action=goto_table=5"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=100, dl_dst=00:00:5E:00:02:03, action=goto_table=5"
# Send L3 to L2 Rewrite Table
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=0, action=goto_table=20"

## Table 5 - L3 Rewrites
# Exclude connected subnets
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=65535, dl_type=0x0800, nw_dst=10.10.10.0/24 actions=goto_table=10"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=65535, dl_type=0x0800, nw_dst=10.10.20.0/24 actions=goto_table=10"
# DNAT
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=100, dl_type=0x0800,  nw_dst=172.16.1.10 actions=mod_nw_dst=10.10.10.2, goto_table=10"
# SNAT
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=100, dl_type=0x0800,  nw_src=10.10.10.2, actions=mod_nw_src=172.16.1.10,  goto_table=10"
# If no rewrite needed, continue to table 10
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=0, actions=goto_table=10"

## Table 10 - IPv4 Routing
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, dl_type=0x0800, nw_dst=10.10.10.0/24, actions=mod_dl_src=00:00:5E:00:02:01, dec_ttl, goto_table=15"
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, dl_type=0x0800, nw_dst=10.10.20.0/24, actions=mod_dl_src=00:00:5E:00:02:02, dec_ttl, goto_table=15"
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, dl_type=0x0800, nw_dst=172.16.1.0/24, actions=mod_dl_src=00:00:5E:00:02:03, dec_ttl, goto_table=15"
# Explicit drop if cannot route
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, priority=0, actions=output:0"

## Table 15 - L3 Forwarding
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=10.10.10.2, actions=mod_dl_dst:00:00:00:00:00:01, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=10.10.20.2, actions=mod_dl_dst:00:00:00:00:00:02, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=10.10.20.4, actions=mod_dl_dst:00:00:00:00:00:04, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=172.16.1.2, actions=mod_dl_dst:00:00:00:00:00:03, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, priority=0, actions=goto_table=20"

## Table 20 - L2 Rewrite
# Go to next table
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=20, priority=0, actions=goto_table=25"

## Table 25 - L2 Forwarding
# Use groups for BUM traffic
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=1, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=1"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=2, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=2"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=3, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=3"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=4, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=2"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:01,actions=output=1"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:02,actions=output=2"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:03,actions=output=3"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:04,actions=output=4"

## Table 105 - ARP Responder
# Respond to ARP for Router Addresses
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105, dl_type=0x0806, nw_dst=10.10.10.1, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[], mod_dl_src:00:00:5E:00:02:01, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000201->NXM_NX_ARP_SHA[], load:0x0a0a0a01->NXM_OF_ARP_SPA[], in_port"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105,  dl_type=0x0806, nw_dst=10.10.20.1, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],  mod_dl_src:00:00:5E:00:02:02, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000202->NXM_NX_ARP_SHA[], load:0xa0a1401->NXM_OF_ARP_SPA[], in_port"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105,  dl_type=0x0806, nw_dst=172.16.1.1, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],  mod_dl_src:00:00:5E:00:02:03, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000203->NXM_NX_ARP_SHA[], load:0xac100101->NXM_OF_ARP_SPA[], in_port"
# Proxy ARP for all floating IPs go below
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105, dl_type=0x0806, nw_dst=172.16.1.10, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[], mod_dl_src:00:00:5E:00:02:03, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000203->NXM_NX_ARP_SHA[], load:0xac10010a->NXM_OF_ARP_SPA[], in_port"
# if we made it here, the arp packet is to be handled as any other regular L2 packet
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105, priority=0, action=resubmit(,20)"

If you'd like to paste these in without comments, you can use this Gist

Testing

We can run a pingall to test this out:

mininet> pingall
*** Ping: testing ping reachability
h1 -> h2 h3 h4
h2 -> h1 X h4
h3 -> X X X
h4 -> h1 h2 X
*** Results: 41% dropped (7/12 received)

hosts 1,2 and 4 can speak to each other. Everything initiated by h3 is dropped (as expected) but h1 can speak to h3 (thanks to NAT). We can test our DNAT and Proxy ARP for Floating IP's using this command:

mininet> h3 ping 172.16.1.10
PING 172.16.1.10 (172.16.1.10) 56(84) bytes of data.
64 bytes from 172.16.1.10: icmp_seq=1 ttl=63 time=1.30 ms
64 bytes from 172.16.1.10: icmp_seq=2 ttl=63 time=0.043 ms
64 bytes from 172.16.1.10: icmp_seq=3 ttl=63 time=0.045 ms
64 bytes from 172.16.1.10: icmp_seq=4 ttl=63 time=0.050 ms
^C
--- 172.16.1.10 ping statistics ---
4 packets transmitted, 4 received, 0% packet loss, time 2999ms
rtt min/avg/max/mdev = 0.043/0.360/1.305/0.545 ms

Cool! It works!

Conclusion

So far we've implemented the nuts and bolts of routing, but we are missing one crucial piece - ICMP handling. Without this useful things like Path MTU Discovery won't work and neither will diagnostics tools like Ping and Traceroute. I think we can do this using a Open vSwitch but first we'll need to add some new NXM fields to ICMP Data and ICMP Checksum and to also make the existing OXM's writable through set_field. I'm going to start talking to the OVS community about this to see if it's possible so watch this space!

@dave_tucker

Helpful Links and Further Reading

ovs-ofctl Man Page Neutron ARP Responder Write Up Address Resolution Protocol OpenFlow 1.3.1 Spec

基於Open vSwitch搭建虛擬路由器

Test Topology

My test topology looks like this:

We have a host in an external network 172.16.1.0/24, one host in an internal network 10.10.10.0/24 and two hosts in another internal network 10.10.20.0/24.

As such, The hosts in the 10.x.x.x range should be able to speak to each other, but should not be able to speak to external hosts.

If you'd like to recreate this topology you can checkout the OpenDaylight OVSDB project source on GitHub and:

 
                   vagrant up mininet
vagrant ssh mininet
cd /vagrant/resources/mininet
sudo mn --custom topo.py --topo l3 
                  

The Pipeline

Our router is implemented using the following pipeline:

Table 0 - Classifier

In this table we work out what traffic is interesting for us before pushing it further along the pipeline

Table 100 - ACL

While we don't use this table today, the idea would be to filter traffic in this table (or series of tables) and to then resubmit to the classifier once we have scrubbed them.

Table 105 - ARP Responder

In this table we use some OVS-Jitsu to take an incoming ARP Request and turn it in to an ARP reply

Table 5 - L3 Rewrite

In this table, we make any L3 modifications we need to before a packet is routed

Table 10 - L3 Routing

The L3 routing table is where the routing magic happens. Here we modify the Source MAC address, Decrement the TTL and push to the L2 tables for forwarding

Table 15 - L3 Forwarding

In the L3 forwarding table we resolve a destination IP address to the correct MAC address for L2 forwarding.

Table 20 - L2 Rewrites

While we aren't using this table in this example, typically we would push/pop any L2 encapsulations here like a VLAN tag or a VXLAN/GRE Tunnel ID.

Table 25 - L2 Forwarding

In this table we do our L2 lookup and forward out the correct port. We also handle L2 BUM traffic here using OpenFlow Groups - no VLANs!

The Flows

To program the flows, paste the following in to mininet:

## Set Bridge to use OpenFlow 1.3
sh ovs-vsctl set Bridge s1 "protocols=OpenFlow13"

## Create Groups
sh ovs-ofctl add-group -OOpenFlow13 s1 group_id=1,type=all,bucket=output:1
sh ovs-ofctl add-group -OOpenFlow13 s1 group_id=2,type=all,bucket=output:2,4
sh ovs-ofctl add-group -OOpenFlow13 s1 group_id=3,type=all,bucket=output:3

## Table 0 - Classifier
# Send ARP to ARP Responder
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=1000, dl_type=0x0806, actions=goto_table=105"
# Send L3 traffic to L3 Rewrite Table
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=100, dl_dst=00:00:5E:00:02:01, action=goto_table=5"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=100, dl_dst=00:00:5E:00:02:02, action=goto_table=5"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=100, dl_dst=00:00:5E:00:02:03, action=goto_table=5"
# Send L3 to L2 Rewrite Table
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=0, priority=0, action=goto_table=20"

## Table 5 - L3 Rewrites
# Exclude connected subnets
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=65535, dl_type=0x0800, nw_dst=10.10.10.0/24 actions=goto_table=10"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=65535, dl_type=0x0800, nw_dst=10.10.20.0/24 actions=goto_table=10"
# DNAT
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=100, dl_type=0x0800,  nw_dst=172.16.1.10 actions=mod_nw_dst=10.10.10.2, goto_table=10"
# SNAT
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=100, dl_type=0x0800,  nw_src=10.10.10.2, actions=mod_nw_src=172.16.1.10,  goto_table=10"
# If no rewrite needed, continue to table 10
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=5, priority=0, actions=goto_table=10"

## Table 10 - IPv4 Routing
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, dl_type=0x0800, nw_dst=10.10.10.0/24, actions=mod_dl_src=00:00:5E:00:02:01, dec_ttl, goto_table=15"
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, dl_type=0x0800, nw_dst=10.10.20.0/24, actions=mod_dl_src=00:00:5E:00:02:02, dec_ttl, goto_table=15"
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, dl_type=0x0800, nw_dst=172.16.1.0/24, actions=mod_dl_src=00:00:5E:00:02:03, dec_ttl, goto_table=15"
# Explicit drop if cannot route
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=10, priority=0, actions=output:0"

## Table 15 - L3 Forwarding
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=10.10.10.2, actions=mod_dl_dst:00:00:00:00:00:01, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=10.10.20.2, actions=mod_dl_dst:00:00:00:00:00:02, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=10.10.20.4, actions=mod_dl_dst:00:00:00:00:00:04, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, dl_type=0x0800, nw_dst=172.16.1.2, actions=mod_dl_dst:00:00:00:00:00:03, goto_table=20"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=15, priority=0, actions=goto_table=20"

## Table 20 - L2 Rewrite
# Go to next table
sh  ovs-ofctl add-flow -OOpenFlow13 s1 "table=20, priority=0, actions=goto_table=25"

## Table 25 - L2 Forwarding
# Use groups for BUM traffic
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=1, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=1"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=2, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=2"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=3, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=3"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, in_port=4, dl_dst=01:00:00:00:00:00/01:00:00:00:00:00, actions=group=2"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:01,actions=output=1"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:02,actions=output=2"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:03,actions=output=3"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=25, dl_dst=00:00:00:00:00:04,actions=output=4"

## Table 105 - ARP Responder
# Respond to ARP for Router Addresses
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105, dl_type=0x0806, nw_dst=10.10.10.1, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[], mod_dl_src:00:00:5E:00:02:01, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000201->NXM_NX_ARP_SHA[], load:0x0a0a0a01->NXM_OF_ARP_SPA[], in_port"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105,  dl_type=0x0806, nw_dst=10.10.20.1, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],  mod_dl_src:00:00:5E:00:02:02, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000202->NXM_NX_ARP_SHA[], load:0xa0a1401->NXM_OF_ARP_SPA[], in_port"
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105,  dl_type=0x0806, nw_dst=172.16.1.1, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],  mod_dl_src:00:00:5E:00:02:03, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000203->NXM_NX_ARP_SHA[], load:0xac100101->NXM_OF_ARP_SPA[], in_port"
# Proxy ARP for all floating IPs go below
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105, dl_type=0x0806, nw_dst=172.16.1.10, actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[], mod_dl_src:00:00:5E:00:02:03, load:0x2->NXM_OF_ARP_OP[], move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[], move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x00005e000203->NXM_NX_ARP_SHA[], load:0xac10010a->NXM_OF_ARP_SPA[], in_port"
# if we made it here, the arp packet is to be handled as any other regular L2 packet
sh ovs-ofctl add-flow -OOpenFlow13 s1 "table=105, priority=0, action=resubmit(,20)"

If you'd like to paste these in without comments, you can use this Gist

Testing

We can run a pingall to test this out:

mininet> pingall
*** Ping: testing ping reachability
h1 -> h2 h3 h4
h2 -> h1 X h4
h3 -> X X X
h4 -> h1 h2 X
*** Results: 41% dropped (7/12 received)

mininet> h3 ping 172.16.1.10
PING 172.16.1.10 (172.16.1.10) 56(84) bytes of data.
64 bytes from 172.16.1.10: icmp_seq=1 ttl=63 time=1.30 ms
64 bytes from 172.16.1.10: icmp_seq=2 ttl=63 time=0.043 ms
64 bytes from 172.16.1.10: icmp_seq=3 ttl=63 time=0.045 ms
64 bytes from 172.16.1.10: icmp_seq=4 ttl=63 time=0.050 ms
^C
--- 172.16.1.10 ping statistics ---
4 packets transmitted, 4 received, 0% packet loss, time 2999ms
rtt min/avg/max/mdev = 0.043/0.360/1.305/0.545 ms

Cool! It works!

Conclusion

@dave_tucker

Helpful Links and Further Reading

ovs-ofctl Man Page Neutron ARP Responder Write Up Address Resolution Protocol OpenFlow 1.3.1 Spec

端口	說明
p0	IP 地址: 192.168.1.100/24 網絡名稱空間: ns0 網絡接口 MAC地址: 66:4e:cc:ae:4d:20 OpenFlow Port Number: 100
p1	IP 地址: 192.168.1.101/24 網絡名稱空間: ns1 網絡接口 MAC地址: 46:54:8a:95:dd:f8 OpenFlow Port Number: 101
p2	IP 地址: 192.168.1.102/24, 網絡名稱空間: ns2 網絡接口 MAC地址: 86:3b:c8:d0:44:10 OpenFlow Port Number: 102

OVS——大雜燴

OVS架構解析--dpdk datapath數據通路

1. 架構說明

1.1. 總體架構

1.2. 核心組件及其關聯關係

1.3. 內部模塊及其關聯關係

2. vswitchd和ovsdb通訊

2.1. 創建鏈接

2.2. 接收數據

2.3. json數據解析

2.4. 更新數據改變標記

3. dpdk加速處理

3.1. ovs dpdk加速配置

3.2. dpdk初始化

4. 數據結構抽象關係

4.1. 分層抽象

4.2. 主要數據結構關係

5. vswitchd啓動

6. 附錄-數據結構查詢

6.1. ofproto_class

6.2. dpif_class

6.3. netdev_class

6.4. bridge

6.5. ofproto

6.6. port

6.7. iface

6.8. ofport

6.9. ofport_dpif

6.10. ofproto_port

6.11. dpif_port

6.12. ofproto_dpif

6.13. dpif_backer

6.14. udpif

6.15. dpif

6.16. dpif_netdev

6.17. dp_netdev

6.18. dp_netdev_port

6.19. netdev

6.20. netdev_dpdk

6.21. dp_netdev_rxq

6.22. netdev_rxq

6.23. netdev_rxq_dpdk

6.24. dp_netdev_pmd_thread

6.25. rxq_poll

6.26. tx_port

OVS 架構

OVS 源碼結構

數據轉發流程

總結

建立一個OVS 交換機

建立的端口信息

使用 ovs-ofctl 建立並測試 OpenFlow 命令

查看 datapath 的信息

屏蔽數據包

修改數據包

重定向數據包

修改數據包的 VLAN Tag

其餘 OpenFlow 經常使用的操做

OVS架構解析--dpdk datapath數據通路

https://blog.csdn.net/chen98765432101/article/details/79835435

1. 架構說明

1.1. 總體架構

1.2. 核心組件及其關聯關係

1.3. 內部模塊及其關聯關係

2. vswitchd和ovsdb通訊

2.1. 創建鏈接

2.2. 接收數據

2.3. json數據解析

2.4. 更新數據改變標記

3. dpdk加速處理

3.1. ovs dpdk加速配置

3.2. dpdk初始化

4. 數據結構抽象關係

4.1. 分層抽象

4.2. 主要數據結構關係

5. vswitchd啓動

6. 附錄-數據結構查詢

6.1. ofproto_class

6.2. dpif_class

6.3. netdev_class