Make overlay driver work without a kv store

Currently overlay driver requires a k/v store to allocate a vxlan id and
add an entry in k/v store for network->vxlanIDs binding. But the overlay
driver should be able to work without a k/v store provided libnetwork
can pass along the vxlanIDs needed for the network, rather than the
driver managing it themselves. Modified the driver to work with vxlanIDs
passed down by libnetwork.

Also made changes in the driver to make use of the gossip layer
available in libnetwork if available.

Signed-off-by: Jana Radhakrishnan <mrjana@docker.com>

Author: mrjana

drivers/overlay/joinleave.go

@@ -3,6 +3,7 @@ package overlay
 import (
 	"fmt"
 	"net"
+	"strings"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/driverapi"
@@ -104,11 +105,55 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 
 	d.peerDbAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac,
 		net.ParseIP(d.bindAddress), true)
+
+	if err := jinfo.AddTableEntry(ovPeerTable, eid, []byte(fmt.Sprintf("%s,%s,%s", ep.addr, ep.mac, d.bindAddress))); err != nil {
+		log.Errorf("overlay: Failed adding table entry to joininfo: %v", err)
+	}
+
 	d.pushLocalEndpointEvent("join", nid, eid)
 
 	return nil
 }
 
+func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key string, value []byte) {
+	if tableName != ovPeerTable {
+		log.Errorf("Unexpected table notification for table %s received", tableName)
+		return
+	}
+
+	eid := key
+	values := strings.Split(string(value), ",")
+	if len(values) < 3 {
+		log.Errorf("Invalid value %s received through event notify", string(value))
+		return
+	}
+
+	addr, err := types.ParseCIDR(values[0])
+	if err != nil {
+		log.Errorf("Invalid peer IP %s received in event notify", values[0])
+		return
+	}
+
+	mac, err := net.ParseMAC(values[1])
+	if err != nil {
+		log.Errorf("Invalid mac %s received in event notify", values[1])
+		return
+	}
+
+	vtep := net.ParseIP(values[2])
+	if vtep == nil {
+		log.Errorf("Invalid VTEP %s received in event notify", values[2])
+		return
+	}
+
+	if etype == driverapi.Delete {
+		d.peerDelete(nid, eid, addr.IP, addr.Mask, mac, vtep, true)
+		return
+	}
+
+	d.peerAdd(nid, eid, addr.IP, addr.Mask, mac, vtep, true)
+}
+
 // Leave method is invoked when a Sandbox detaches from an endpoint.
 func (d *driver) Leave(nid, eid string) error {
 	if err := validateID(nid, eid); err != nil {

drivers/overlay/ov_network.go

@@ -6,13 +6,15 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/datastore"
 	"github.com/docker/libnetwork/driverapi"
+	"github.com/docker/libnetwork/netlabel"
 	"github.com/docker/libnetwork/netutils"
 	"github.com/docker/libnetwork/osl"
 	"github.com/docker/libnetwork/resolvconf"
@@ -67,9 +69,6 @@ func (d *driver) NetworkFree(id string) error {
 	return types.NotImplementedErrorf("not implemented")
 }
 
-func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key string, value []byte) {
-}
-
 func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
 	if id == "" {
 		return fmt.Errorf("invalid network id")
@@ -92,21 +91,54 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d
 		subnets:   []*subnet{},
 	}
 
-	for _, ipd := range ipV4Data {
+	vnis := make([]uint32, 0, len(ipV4Data))
+	if gval, ok := option[netlabel.GenericData]; ok {
+		optMap := gval.(map[string]string)
+		if val, ok := optMap[netlabel.OverlayVxlanIDList]; ok {
+			logrus.Debugf("overlay: Received vxlan IDs: %s", val)
+			vniStrings := strings.Split(val, ",")
+			for _, vniStr := range vniStrings {
+				vni, err := strconv.Atoi(vniStr)
+				if err != nil {
+					return fmt.Errorf("invalid vxlan id value %q passed", vniStr)
+				}
+
+				vnis = append(vnis, uint32(vni))
+			}
+		}
+	}
+
+	// If we are getting vnis from libnetwork, either we get for
+	// all subnets or none.
+	if len(vnis) != 0 && len(vnis) < len(ipV4Data) {
+		return fmt.Errorf("insufficient vnis(%d) passed to overlay", len(vnis))
+	}
+
+	for i, ipd := range ipV4Data {
 		s := &subnet{
 			subnetIP: ipd.Pool,
 			gwIP:     ipd.Gateway,
 			once:     &sync.Once{},
 		}
+
+		if len(vnis) != 0 {
+			s.vni = vnis[i]
+		}
+
 		n.subnets = append(n.subnets, s)
 	}
 
 	if err := n.writeToStore(); err != nil {
 		return fmt.Errorf("failed to update data store for network %v: %v", n.id, err)
 	}
 
-	d.addNetwork(n)
+	if nInfo != nil {
+		if err := nInfo.TableEventRegister(ovPeerTable); err != nil {
+			return err
+		}
+	}
 
+	d.addNetwork(n)
 	return nil
 }
 
@@ -255,11 +287,21 @@ func setHostMode() {
 }
 
 func (n *network) generateVxlanName(s *subnet) string {
-	return "vx-" + fmt.Sprintf("%06x", n.vxlanID(s)) + "-" + n.id[:5]
+	id := n.id
+	if len(n.id) > 5 {
+		id = n.id[:5]
+	}
+
+	return "vx-" + fmt.Sprintf("%06x", n.vxlanID(s)) + "-" + id
 }
 
 func (n *network) generateBridgeName(s *subnet) string {
-	return "ov-" + fmt.Sprintf("%06x", n.vxlanID(s)) + "-" + n.id[:5]
+	id := n.id
+	if len(n.id) > 5 {
+		id = n.id[:5]
+	}
+
+	return "ov-" + fmt.Sprintf("%06x", n.vxlanID(s)) + "-" + id
 }
 
 func isOverlap(nw *net.IPNet) bool {
@@ -587,32 +629,38 @@ func (n *network) DataScope() string {
 }
 
 func (n *network) writeToStore() error {
+	if n.driver.store == nil {
+		return nil
+	}
+
 	return n.driver.store.PutObjectAtomic(n)
 }
 
 func (n *network) releaseVxlanID() error {
-	if n.driver.store == nil {
-		return fmt.Errorf("no datastore configured. cannot release vxlan id")
-	}
-
 	if len(n.subnets) == 0 {
 		return nil
 	}
 
-	if err := n.driver.store.DeleteObjectAtomic(n); err != nil {
-		if err == datastore.ErrKeyModified || err == datastore.ErrKeyNotFound {
-			// In both the above cases we can safely assume that the key has been removed by some other
-			// instance and so simply get out of here
-			return nil
-		}
+	if n.driver.store != nil {
+		if err := n.driver.store.DeleteObjectAtomic(n); err != nil {
+			if err == datastore.ErrKeyModified || err == datastore.ErrKeyNotFound {
+				// In both the above cases we can safely assume that the key has been removed by some other
+				// instance and so simply get out of here
+				return nil
+			}
 
-		return fmt.Errorf("failed to delete network to vxlan id map: %v", err)
+			return fmt.Errorf("failed to delete network to vxlan id map: %v", err)
+		}
 	}
 
 	for _, s := range n.subnets {
-		n.driver.vxlanIdm.Release(uint64(n.vxlanID(s)))
+		if n.driver.vxlanIdm != nil {
+			n.driver.vxlanIdm.Release(uint64(n.vxlanID(s)))
+		}
+
 		n.setVxlanID(s, 0)
 	}
+
 	return nil
 }
 
@@ -623,7 +671,7 @@ func (n *network) obtainVxlanID(s *subnet) error {
 	}
 
 	if n.driver.store == nil {
-		return fmt.Errorf("no datastore configured. cannot obtain vxlan id")
+		return fmt.Errorf("no valid vxlan id and no datastore configured, cannot obtain vxlan id")
 	}
 
 	for {

drivers/overlay/overlay.go

@@ -88,7 +88,7 @@ func Fini(drv driverapi.Driver) {
 
 func (d *driver) configure() error {
 	if d.store == nil {
-		return types.NoServiceErrorf("datastore is not available")
+		return nil
 	}
 
 	if d.vxlanIdm == nil {

drivers/overlay/overlay_test.go

@@ -8,7 +8,6 @@ import (
 	"github.com/docker/libnetwork/discoverapi"
 	"github.com/docker/libnetwork/driverapi"
 	_ "github.com/docker/libnetwork/testutils"
-	"github.com/docker/libnetwork/types"
 )
 
 type driverTester struct {
@@ -24,14 +23,6 @@ func setupDriver(t *testing.T) *driverTester {
 		t.Fatal(err)
 	}
 
-	err := dt.d.configure()
-	if err == nil {
-		t.Fatalf("Failed to detect nil store")
-	}
-	if _, ok := err.(types.NoServiceError); !ok {
-		t.Fatalf("Unexpected error type: %v", err)
-	}
-
 	iface, err := net.InterfaceByName("eth0")
 	if err != nil {
 		t.Fatal(err)
@@ -93,23 +84,6 @@ func TestOverlayFiniWithoutConfig(t *testing.T) {
 	cleanupDriver(t, dt)
 }
 
-func TestOverlayNilConfig(t *testing.T) {
-	dt := &driverTester{t: t}
-	if err := Init(dt, nil); err != nil {
-		t.Fatal(err)
-	}
-
-	err := dt.d.configure()
-	if err == nil {
-		t.Fatalf("Failed to detect nil store")
-	}
-	if _, ok := err.(types.NoServiceError); !ok {
-		t.Fatalf("Unexpected error type: %v", err)
-	}
-
-	cleanupDriver(t, dt)
-}
-
 func TestOverlayConfig(t *testing.T) {
 	dt := setupDriver(t)
 

drivers/overlay/peerdb.go

@@ -7,6 +7,8 @@ import (
 	"syscall"
 )
 
+const ovPeerTable = "overlay_peer_table"
+
 type peerKey struct {
 	peerIP  net.IP
 	peerMac net.HardwareAddr