Developing lightweight computation at the DSG edge

Commit 8ff81163 authored by Roger Pueyo Centelles's avatar Roger Pueyo Centelles
Browse files

[fetch] Fix loading a CNML file with nodes already in AntidoteDB



Loading a CNML file which contained devices already known to AntidoteDB
made the update process perform unneded DB operations and loops which
terribly loaded the system.
Signed-off-by: default avatarRoger Pueyo Centelles <roger.pueyo@guifi.net>
parent 40511d81
......@@ -8,6 +8,7 @@ import (
"fmt"
"io/ioutil"
"os"
"sort"
"strconv"
"strings"
......@@ -131,9 +132,7 @@ func main() {
dJFile.WriteString(string(jdevices))
}
// TODO: remove devsadds (devsIpv4sGraphs does the thing)
// var devsadds []cnml.DeviceIpv4Adddresses
var devsIpv4sGraphs []cnml.DeviceIpv4sGraphserver
var cnmlDevsIpv4sGraphs []cnml.DeviceIpv4sGraphserver
// Export only the bare minimum information needed for the monitoring code
for _, v := range allDevices {
......@@ -155,7 +154,7 @@ func main() {
// address) and are either in working or in testing mode
if len(devadd.Addresses) > 0 && (v.Status == "Working" || v.Status == "Testing") {
// devsadds = append(devsadds, devadd)
devsIpv4sGraphs = append(devsIpv4sGraphs, devaddsrv)
cnmlDevsIpv4sGraphs = append(cnmlDevsIpv4sGraphs, devaddsrv)
jdevadd, err := json.Marshal(devadd)
errCheck(err, "")
......@@ -180,6 +179,8 @@ func main() {
// Read current AntidoteDB devices
antidoteDevices := apidote.ReadDevicesFromAntidote()
var oldAntidoteDevices []cnml.DeviceIpv4sGraphserver
var keepAntidoteDevices []cnml.DeviceIpv4sGraphserver
// Export devices and their addresses to AntidoteDB
antidoteAddDeviceSuccess := 0
......@@ -195,44 +196,42 @@ func main() {
antidoteDeleteGraphserverSuccess := 0
antidoteDeleteGraphserverFail := 0
// Remove from AntidoteDB those devices that are not in the CNML file anymore
for k, v := range antidoteDevices {
if !isDeviceIpv4sGraphserverInArray(v, devsIpv4sGraphs) {
// Remove IPv4 addresses of the removed device
for _, w := range readDeviceIpv4sGraphserverFromAntidote(v).Addresses {
if removeIpv4addressFromDeviceIpv4sGraphserver(v, w) {
antidoteDeleteAddressSuccess++
} else {
antidoteDeleteAddressFail++
}
}
//Remove the Graphserver
if removeGraphserverFromDeviceIpv4sGraphserver(v) {
antidoteDeleteGraphserverSuccess++
} else {
antidoteDeleteGraphserverFail++
}
// Remove the device from the list
if removeDeviceFromAntidote(v.ID) {
antidoteDeleteDeviceSuccess++
} else {
antidoteDeleteDeviceFail++
}
// If the device is in the CNML file, check for old IPv4 addresses no longer in use and remove them
// Check AntidoteDB for old devices not in the current CNML file anymore
for _, v := range antidoteDevices {
if !isDeviceIpv4sGraphserverInArray(v, cnmlDevsIpv4sGraphs) {
// Save old devices to later remove them
oldAntidoteDevices = append(oldAntidoteDevices, v)
} else {
// Save devices to keep to later check if any changes have appeared
keepAntidoteDevices = append(keepAntidoteDevices, v)
}
}
for _, w := range devsIpv4sGraphs {
for _, x := range v.Addresses {
if !isIPv4addressInDeviceIpv4sGraphserversArray(w, x) {
if removeIpv4addressFromDeviceIpv4sGraphserver(w, x) {
antidoteDeleteAddressSuccess++
} else {
antidoteDeleteAddressFail++
}
}
}
fmt.Println("")
// Remove old devices from AntidoteDB not in the CNML file anymore
for k, v := range oldAntidoteDevices {
// Remove the device's graphserver from AntidoteDB
if removeGraphserverFromDeviceIpv4sGraphserver(v) {
antidoteDeleteGraphserverSuccess++
} else {
antidoteDeleteGraphserverFail++
}
// Remove the device's IPv4 addresses from AntidoteDB
for _, w := range v.Addresses {
if removeIpv4addressFromDeviceIpv4sGraphserver(v, w) {
antidoteDeleteAddressSuccess++
} else {
antidoteDeleteAddressFail++
}
}
// Remove the device from AntidoteDB
if removeDeviceFromAntidote(v.ID) {
antidoteDeleteDeviceSuccess++
} else {
antidoteDeleteDeviceFail++
}
if k == 0 {
fmt.Printf("%d devices removed from AntidoteDB (%d success, %d fail)\n", antidoteDeleteDeviceSuccess+antidoteDeleteDeviceFail, antidoteDeleteDeviceSuccess, antidoteDeleteDeviceFail)
......@@ -245,15 +244,21 @@ func main() {
}
}
fmt.Printf("\033[3A\r%d devices removed from AntidoteDB (%d success, %d fail)\n", antidoteDeleteDeviceSuccess+antidoteDeleteDeviceFail, antidoteDeleteDeviceSuccess, antidoteDeleteDeviceFail)
if len(oldAntidoteDevices) > 0 {
fmt.Printf("\033[3A\r%d devices removed from AntidoteDB (%d success, %d fail)\n", antidoteDeleteDeviceSuccess+antidoteDeleteDeviceFail, antidoteDeleteDeviceSuccess, antidoteDeleteDeviceFail)
} else {
fmt.Printf("\r%d devices removed from AntidoteDB (%d success, %d fail)\n", antidoteDeleteDeviceSuccess+antidoteDeleteDeviceFail, antidoteDeleteDeviceSuccess, antidoteDeleteDeviceFail)
}
fmt.Printf("\r%d graphservers removed from AntidoteDB (%d success, %d fail)\n", antidoteDeleteGraphserverSuccess+antidoteDeleteGraphserverFail, antidoteDeleteGraphserverSuccess, antidoteDeleteGraphserverFail)
fmt.Printf("\r%d IPv4 addresses removed from AntidoteDB (%d success, %d fail)\n", antidoteDeleteAddressSuccess+antidoteDeleteAddressFail, antidoteDeleteAddressSuccess, antidoteDeleteAddressFail)
// Add to AntidoteDB those devices that are new in the CNML file
for k, v := range devsIpv4sGraphs {
fmt.Println("")
// Add or update those devices that are in the CNML file
for k, v := range cnmlDevsIpv4sGraphs {
// Check if the device is in Antidote already and add it if missing
saveDevFail := false
if !isDeviceIpv4sGraphserverInArray(v, antidoteDevices) && len(v.Addresses) > 0 {
if !isDeviceIpv4sGraphserverInArray(v, keepAntidoteDevices) && len(v.Addresses) > 0 {
saveDevFail := false
// Add the device to the devices set
if saveDeviceIpv4sGraphserverToAntidote(v) {
antidoteAddDeviceSuccess++
......@@ -261,26 +266,57 @@ func main() {
antidoteAddDeviceFail++
saveDevFail = true
}
}
saveGraphserverFail := false
if !saveDevFail {
// Save the default graph server
if saveGraphserverToDeviceIpv4sGraphserverToAntidote(v) {
antidoteAddGraphserverSuccess++
} else {
antidoteAddGraphserverFail++
saveGraphserverFail = true
}
if !saveDevFail {
// Save the default graph server
if saveGraphserverToDeviceIpv4sGraphserverToAntidote(v) {
antidoteAddGraphserverSuccess++
} else {
antidoteAddGraphserverFail++
}
}
if !saveDevFail && !saveGraphserverFail {
// Iterate through all the device's addresses and add them if new
for _, w := range v.Addresses {
if saveIpv4addressToDeviceIpv4sGraphserverToAntidote(v, w) {
antidoteAddAddressSuccess++
// Iterate through all the device's addresses and add them if new
for _, w := range v.Addresses {
if saveIpv4addressToDeviceIpv4sGraphserverToAntidote(v, w) {
antidoteAddAddressSuccess++
} else {
antidoteAddAddressFail++
}
}
}
} else {
// The device is in AntidoteDB; check if it has changed
if v.GraphServer != keepAntidoteDevices[posDeviceInSlice(v, keepAntidoteDevices)].GraphServer {
// Save the new default graph server
if saveGraphserverToDeviceIpv4sGraphserverToAntidote(v) {
antidoteAddGraphserverSuccess++
} else {
antidoteAddAddressFail++
antidoteAddGraphserverFail++
}
}
ips1 := v.Addresses
ips2 := keepAntidoteDevices[posDeviceInSlice(v, keepAntidoteDevices)].Addresses
sort.Strings(ips1)
sort.Strings(ips2)
if !compareStringArrays(ips1, ips2) {
for _, w := range keepAntidoteDevices[posDeviceInSlice(v, keepAntidoteDevices)].Addresses {
if removeIpv4addressFromDeviceIpv4sGraphserver(v, w) {
antidoteDeleteAddressSuccess++
println("A")
} else {
antidoteDeleteAddressFail++
}
}
// Iterate through all the device's addresses and add them if new
for _, w := range v.Addresses {
if saveIpv4addressToDeviceIpv4sGraphserverToAntidote(v, w) {
antidoteAddAddressSuccess++
} else {
antidoteAddAddressFail++
}
}
}
}
......@@ -295,16 +331,6 @@ func main() {
fmt.Printf("\r%d IPv4 addresses added or updated to AntidoteDB (%d success, %d fail)\n", antidoteAddAddressSuccess+antidoteAddAddressFail, antidoteAddAddressSuccess, antidoteAddAddressFail)
}
}
fmt.Printf("\033[3A\r%d devices added to AntidoteDB (%d success, %d fail)\n", antidoteAddDeviceSuccess+antidoteAddDeviceFail, antidoteAddDeviceSuccess, antidoteAddDeviceFail)
fmt.Printf("\r%d graphservers added or updated to AntidoteDB (%d success, %d fail)\n", antidoteAddGraphserverSuccess+antidoteAddGraphserverFail, antidoteAddGraphserverSuccess, antidoteAddGraphserverFail)
fmt.Printf("\r%d IPv4 addresses added or updated to AntidoteDB (%d success, %d fail)\n", antidoteAddAddressSuccess+antidoteAddAddressFail, antidoteAddAddressSuccess, antidoteAddAddressFail)
if antidoteAddDeviceSuccess+antidoteAddDeviceFail == 0 {
// fmt.Printf("\n")
}
// fmt.Printf("\033[2A\r%d devices added to AntidoteDB (%d success, %d failc)\n", antidoteAddDeviceSuccess+antidoteAddDeviceFail, antidoteAddDeviceSuccess, antidoteAddDeviceFail)
// fmt.Printf("\r%d IPv4 addresses added or updated to AntidoteDB (%d success, %d faild)\n", antidoteAddAddressSuccess+antidoteAddAddressFail, antidoteAddAddressSuccess, antidoteAddAddressFail)
}
// Get an array with all the nodes in a zone and its subzones, recursively
......@@ -572,6 +598,19 @@ func removeIpv4addressFromDeviceIpv4sGraphserver(device cnml.DeviceIpv4sGraphser
return apidote.AntidoteRemoveItemFromSetInBucket(fmt.Sprintf("device-%d", device.ID), "ipv4s", address)
}
// Compare two arrays of strings
func compareStringArrays(array1 []string, array2 []string) bool {
if len(array1) == len(array2) {
for k := range array1 {
if array1[k] != array2[k] {
return false
}
}
return true
}
return false
}
// Check if an int item is in an ints array
func isDeviceIPv4AddressesInArray(device cnml.DeviceIpv4Adddresses, devices []cnml.DeviceIpv4Adddresses) bool {
for _, v := range devices {
......@@ -610,6 +649,15 @@ func isIPv4addressInDeviceIpv4sGraphserversArray(device cnml.DeviceIpv4sGraphser
return false
}
func posDeviceInSlice(device cnml.DeviceIpv4sGraphserver, slice []cnml.DeviceIpv4sGraphserver) int {
for k, v := range slice {
if device.ID == v.ID {
return k
}
}
return -1
}
// Panic on error
func errCheck(err error, message string) {
if err != nil {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment