I used 2x MikroTik RB333 to utilize both links simultaneously, one at each end. I specifically used RouterOS v3.0rc13 because there is a random disconnection issue (every few minutes/hours the bonded link will be disconnected for a few seconds and resume without any signs of symptoms anywhere else) with later versions of RouterOS v3.x (though I haven’t tried RouterOS v4.x).

This configuration also doesn’t give you a fully-working auto fail-over, in case one of the links is broken in the middle — since link state doesn’t change as mentioned in the post (though if it’s physically dead, the auto fail-over will work).

I used ARP detection to check if the links are both up but apparently it didn’t work as expected. So every time there is a broken link, I will disconnect the broken link from the RB333 manually. This way the RB333 will detect the link state change from the disconnected link as down and force all packets to go through the other link. If you don’t disconnect the broken link manually (disabling the ether interface of the broken link is acceptable) from the RB333, it will route 50% of the packets through the broken link (since it doesn’t know the broken link is in fact broken) and you will start seeing major packet losses.

/interface bridge
add admin-mac=00:00:00:00:00:00 ageing-time=5m arp=enabled auto-mac=yes \
comment=”" disabled=no forward-delay=15s max-message-age=20s mtu=1500 \
name=”bridge1″ priority=0×8000 protocol-mode=none transmit-hold-count=6

/interface ethernet
set 0 arp=enabled auto-negotiation=yes comment=”" disabled=no full-duplex=yes \
mac-address=00:0C:42:1D:1E:1B mtu=1500 name=”ether1″ speed=100Mbps
set 1 arp=enabled auto-negotiation=yes comment=”" disabled=no full-duplex=yes \
mac-address=00:0C:42:1D:1E:1C mtu=1500 name=”ether2″ speed=100Mbps
set 2 arp=enabled auto-negotiation=yes comment=”" disabled=no full-duplex=yes \
mac-address=00:0C:42:1D:1E:1D mtu=1500 name=”ether3″ speed=100Mbps

/interface eoip
add arp=enabled comment=”" disabled=yes mac-address=FE:6E:99:E5:DB:2C mtu=1500 \
name=”eoip-tunnel1″ remote-address=172.16.1.1 tunnel-id=1
add arp=enabled comment=”" disabled=yes mac-address=FE:E8:5A:6D:5B:70 mtu=1500 \
name=”eoip-tunnel2″ remote-address=172.16.2.1 tunnel-id=2

/interface bonding
add arp=enabled arp-interval=100ms arp-ip-targets=172.16.0.1 comment=”" \
disabled=no down-delay=0s lacp-rate=30secs link-monitoring=arp \
mii-interval=100ms mode=balance-rr mtu=1500 name=”bonding1″ primary=none \
slaves=ether2,ether3 up-delay=0s

/ip address
add address=10.255.255.101/24 broadcast=10.255.255.255 comment=”" disabled=no \
interface=bridge1 network=10.255.255.0
add address=172.16.0.2/24 broadcast=172.16.0.255 comment=”" disabled=no \
interface=bonding1 network=172.16.0.0
add address=172.16.1.2/24 broadcast=172.16.1.255 comment=”" disabled=yes \
interface=ether2 network=172.16.1.0
add address=172.16.2.2/24 broadcast=172.16.2.255 comment=”" disabled=yes \
interface=ether3 network=172.16.2.0

/ip dns
set allow-remote-requests=no cache-max-ttl=1w cache-size=2048KiB \
primary-dns=208.67.222.222 secondary-dns=208.67.220.220

/ip route
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=10.255.255.254 \
scope=255 target-scope=10

/system identity
set name=”RB333-Sunter”

/system ntp client
set enabled=yes mode=unicast primary-ntp=202.169.237.2 secondary-ntp=202.169.224.16

/interface bridge
add admin-mac=00:00:00:00:00:00 ageing-time=5m arp=enabled auto-mac=yes \
comment=”" disabled=no forward-delay=15s max-message-age=20s mtu=1500 \
name=”bridge1″ priority=0×8000 protocol-mode=none transmit-hold-count=6

/interface ethernet
set 0 arp=enabled auto-negotiation=yes comment=”" disabled=no full-duplex=yes \
mac-address=00:0C:42:1C:9B:B1 mtu=1500 name=”ether1″ speed=100Mbps
set 1 arp=enabled auto-negotiation=yes comment=”" disabled=no full-duplex=yes \
mac-address=00:0C:42:1C:9B:B2 mtu=1500 name=”ether2″ speed=100Mbps
set 2 arp=enabled auto-negotiation=yes comment=”" disabled=no full-duplex=yes \
mac-address=00:0C:42:1C:9B:B3 mtu=1500 name=”ether3″ speed=100Mbps

/interface eoip
add arp=enabled comment=”" disabled=yes mac-address=FE:61:49:D3:D4:4A mtu=1500 \
name=”eoip-tunnel1″ remote-address=172.16.1.2 tunnel-id=1
add arp=enabled comment=”" disabled=yes mac-address=FE:F6:DF:A4:78:24 mtu=1500 \
name=”eoip-tunnel2″ remote-address=172.16.2.2 tunnel-id=2

/interface bonding
add arp=enabled arp-interval=100ms arp-ip-targets=172.16.0.2 comment=”" \
disabled=no down-delay=0s lacp-rate=30secs link-monitoring=arp \
mii-interval=100ms mode=balance-rr mtu=1500 name=”bonding1″ primary=none \
slaves=ether2,ether3 up-delay=0s

/interface bridge port
add bridge=bridge1 comment=”" disabled=no edge=auto external-fdb=auto \
horizon=none interface=ether1 path-cost=10 point-to-point=auto \
priority=0×80
add bridge=bridge1 comment=”" disabled=no edge=auto external-fdb=auto \
horizon=none interface=bonding1 path-cost=10 point-to-point=auto \
priority=0×80

/ip address
add address=10.255.255.100/24 broadcast=10.255.255.255 comment=”" disabled=no \
interface=bridge1 network=10.255.255.0
add address=172.16.0.1/24 broadcast=172.16.0.255 comment=”" disabled=no \
interface=bonding1 network=172.16.0.0
add address=172.16.1.1/24 broadcast=172.16.1.255 comment=”" disabled=yes \
interface=ether2 network=172.16.1.0
add address=172.16.2.1/24 broadcast=172.16.2.255 comment=”" disabled=yes \
interface=ether3 network=172.16.2.0

/ip dns
set allow-remote-requests=no cache-max-ttl=1w cache-size=2048KiB \
primary-dns=208.67.222.222 secondary-dns=208.67.220.220

/ip route
add disabled=no distance=1 dst-address=0.0.0.0/0 gateway=10.255.255.254 \
scope=255 target-scope=10

/system identity
set name=”RB333-IDC”

/system ntp client
set enabled=yes mode=unicast primary-ntp=202.169.237.2 secondary-ntp=202.169.224.16