Developing lightweight computation at the DSG edge

Commit 5f6f8c67 authored by Roger Pueyo Centelles's avatar Roger Pueyo Centelles
Browse files

Save graphserver to Antidote


Signed-off-by: default avatarRoger Pueyo Centelles <roger.pueyo@guifi.net>
parent 369eb4e9
......@@ -52,7 +52,7 @@ func AntidoteReadItemsFromSetInBucket(bucket string, set string) []string {
return items
}
// Antidote: remove an item from a set in a bucket
// AntidoteRemoveItemFromSetInBucket removes an item from a set in a bucket
func AntidoteRemoveItemFromSetInBucket(bucket string, set string, item string) bool {
request := fmt.Sprintf("http://localhost:3000/set/remove/%s/%s/%s", bucket, set, item)
response, err := http.Get(request)
......@@ -73,14 +73,14 @@ func AntidoteRemoveItemFromSetInBucket(bucket string, set string, item string) b
return false
}
// Read all the devices from Antidote using João's REST server
func ReadDevicesFromAntidote() []cnml.DeviceIpv4Adddresses {
var devices []cnml.DeviceIpv4Adddresses
// ReadDevicesFromAntidote read all the devices from Antidote using João's REST server
func ReadDevicesFromAntidote() []cnml.DeviceIpv4sGraphserver {
var devices []cnml.DeviceIpv4sGraphserver
for _, v := range AntidoteReadItemsFromSetInBucket("guifi", "devices") {
thisDevID, err := strconv.Atoi(v)
errCheck(err)
var thisDev cnml.DeviceIpv4Adddresses
var thisDev cnml.DeviceIpv4sGraphserver
thisDev.ID = thisDevID
devices = append(devices, thisDev)
}
......
......@@ -122,8 +122,8 @@ type DeviceAddressesServer struct {
GraphServer int
}
// The DeviceAddresses lists the IPv4 addresses a GraphServer is assigned
type DeviceAddresses struct {
// The DeviceIpv4sGraphserver lists the IPv4 addresses a GraphServer is assigned
type DeviceIpv4sGraphserver struct {
ID int `json:"Id"`
Addresses []string `json:"Addresses"`
GraphServer int `json:"GraphServer"`
......
......@@ -429,7 +429,7 @@ func reassignDevs() {
}
//Create a list with all the unassigned devices that are assigned via web to this monitor
var assignweb []cnml.DeviceIpv4Adddresses
var assignweb []cnml.DeviceIpv4sGraphserver
// for _, v := range unassigned {
// // if fmt.Sprint(v.GraphServer) == ID {
// // assignweb = append(assignweb, v)
......
......@@ -8,6 +8,8 @@ import (
"fmt"
"io/ioutil"
"os"
"strconv"
"strings"
"lightkone.guifi.net/uc-monitor-go-test/apidote"
"lightkone.guifi.net/uc-monitor-go-test/cnml"
......@@ -16,14 +18,36 @@ import (
//Default settings and descriptions
var cnmlFile = "../assets/cnml/upc.xml"
var cnmlFileHelp = "Filename of the XML file containing the CNML information of a zone"
var outDir = "/tmp/gmonitor2"
var outDirHelp = "Directory name where the output files will be saved"
var nodesFile = "nodes.json"
var nodesFileHelp = "Filename where the nodes list in JSON format will be saved"
var devicesFile = "devices.json"
var devicesFileHelp = "Filename where the devices list in JSON format will be saved"
func initialize() {
//Define and parse command line flags
cnmlFilePtr := flag.String("cnml_file", cnmlFile, cnmlFileHelp)
outDirPtr := flag.String("out_dir", outDir, outDirHelp)
nodesFilePtr := flag.String("nodes_file", nodesFile, nodesFileHelp)
devicesFilePtr := flag.String("devices_file", devicesFile, devicesFileHelp)
flag.Parse()
cnmlFile = *cnmlFilePtr
outDir = *outDirPtr
nodesFile = *nodesFilePtr
devicesFile = *devicesFilePtr
// Add a trailing slash to outDir, if missing
if outDir[len(outDir)-1:] != "/" {
outDir = strings.Join([]string{outDir, "/"}, "")
}
// Add path to filenames
nodesFile = strings.Join([]string{outDir, nodesFile}, "")
devicesFile = strings.Join([]string{outDir, devicesFile}, "")
}
......@@ -31,75 +55,90 @@ func main() {
initialize()
// Open the CNML file to be parsed
xmlFile, err := os.Open(cnmlFile)
if err != nil {
fmt.Println("Error opening file:", err)
return
}
errCheck(err, fmt.Sprintf("Error opening file %s:", cnmlFile))
defer xmlFile.Close()
b, _ := ioutil.ReadAll(xmlFile)
//Read the whole CNML file and save it to readFile
readFile, err := ioutil.ReadAll(xmlFile)
errCheck(err, fmt.Sprintf("Error reading file %s:", cnmlFile))
// Unmarshal the read CNML file to a CNML data structure
var GuifiCnml cnml.Cnml
err = xml.Unmarshal(b, &GuifiCnml)
errCheck(err)
err = xml.Unmarshal(readFile, &GuifiCnml)
errCheck(err, fmt.Sprintf("Error parsing XML data in file %s:", cnmlFile))
// Propagate graph servers top-down to those nodes and devices that don't
// have a specific server assigned (i.e., that inheir their parents' one)
GuifiCnml.Network.Zone = propagateGraphServerInZone(GuifiCnml.Network.Zone)
var allnodes = getNodesInZone(GuifiCnml.Network.Zone)
var alldevices = getDevicesInZone(GuifiCnml.Network.Zone)
// Get an array with all the nodes in the CNML
var allNodes = getNodesInZone(GuifiCnml.Network.Zone)
// Get an array with all the devices in the CNML
var allDevices = getDevicesInZone(GuifiCnml.Network.Zone)
fmt.Println(len(allNodes), "nodes read from", xmlFile.Name())
fmt.Println(len(allDevices), "devices read from", xmlFile.Name())
fmt.Println(len(allnodes), "nodes read from", xmlFile.Name())
fmt.Println(len(alldevices), "devices read from", xmlFile.Name())
// Create the output directory
err = os.MkdirAll(outDir, 0755)
errCheck(err, fmt.Sprintf("Error creating working dir %s:", outDir))
err = os.MkdirAll("/tmp/gmonitor2", 0755)
errCheck(err)
nfile, err := os.Create("/tmp/gmonitor2/nodes.json")
errCheck(err)
defer nfile.Close()
// Create the nodes JSON output file
// TODO: Check if really needed
nJFile, err := os.Create(nodesFile)
errCheck(err, fmt.Sprintf("Error creating working dir %s:", outDir))
defer nJFile.Close()
dfile, err := os.Create("/tmp/gmonitor2/devices.json")
errCheck(err)
defer dfile.Close()
// Create the devices JSON output file
dJFile, err := os.Create(devicesFile)
errCheck(err, "")
defer dJFile.Close()
// TODO: Check if really needed
dafile, err := os.Create("/tmp/gmonitor2/devs_old.json")
errCheck(err)
errCheck(err, "")
defer dafile.Close()
dasfile, err := os.Create("/tmp/gmonitor2/devs.json")
errCheck(err)
errCheck(err, "")
defer dasfile.Close()
daChecksumFile, err := os.Create("/tmp/gmonitor2/devs_old.checksum")
errCheck(err)
errCheck(err, "")
defer daChecksumFile.Close()
dasChecksumFile, err := os.Create("/tmp/gmonitor2/devs.checksum")
errCheck(err)
errCheck(err, "")
defer dasChecksumFile.Close()
for _, v := range allnodes {
// Save the whole nodes information to a JSON file
for _, v := range allNodes {
jnodes, err := json.Marshal(v)
errCheck(err)
errCheck(err, "")
//fmt.Println(string(jnode))
nfile.WriteString(string(jnodes))
nJFile.WriteString(string(jnodes))
}
for _, v := range alldevices {
// Save the whole devices information to a JSON file
for _, v := range allDevices {
jdevices, err := json.Marshal(v)
errCheck(err)
errCheck(err, "")
//fmt.Println(string(jdevices))
dfile.WriteString(string(jdevices))
dJFile.WriteString(string(jdevices))
}
var devsadds []cnml.DeviceIpv4Adddresses
var devsaddssrvs []cnml.DeviceAddressesServer
// TODO: remove devsadds (devsIpv4sGraphs does the thing)
// var devsadds []cnml.DeviceIpv4Adddresses
var devsIpv4sGraphs []cnml.DeviceIpv4sGraphserver
// Export only the bare minimum information needed for the monitoring code
for _, v := range alldevices {
for _, v := range allDevices {
var devadd cnml.DeviceIpv4Adddresses
var devaddsrv cnml.DeviceAddressesServer
var devaddsrv cnml.DeviceIpv4sGraphserver
devadd.ID = v.ID
devaddsrv.ID = v.ID
......@@ -115,14 +154,14 @@ func main() {
// Save only those devices which are addressable (i.e. have an IPv4
// address) and are either in working or in testing mode
if len(devadd.Addresses) > 0 && (v.Status == "Working" || v.Status == "Testing") {
devsadds = append(devsadds, devadd)
devsaddssrvs = append(devsaddssrvs, devaddsrv)
// devsadds = append(devsadds, devadd)
devsIpv4sGraphs = append(devsIpv4sGraphs, devaddsrv)
jdevadd, err := json.Marshal(devadd)
errCheck(err)
errCheck(err, "")
jdevaddsrv, err := json.Marshal(devaddsrv)
errCheck(err)
errCheck(err, "")
// fmt.Println(string(jdevadd))
dafile.WriteString(string(jdevadd))
......@@ -132,10 +171,10 @@ func main() {
}
}
daChecksumFile.WriteString(fmt.Sprintf("%x\n", sha256.Sum256([]byte(fmt.Sprintf("%+v", devsadds)))))
dasChecksumFile.WriteString(fmt.Sprintf("%x\n", sha256.Sum256([]byte(fmt.Sprintf("%+v", devsaddssrvs)))))
//daChecksumFile.WriteString(fmt.Sprintf("%x\n", sha256.Sum256([]byte(fmt.Sprintf("%+v", devsadds)))))
dasChecksumFile.WriteString(fmt.Sprintf("%x\n", sha256.Sum256([]byte(fmt.Sprintf("%+v", devsIpv4sGraphs)))))
fmt.Println(len(devsaddssrvs), "devices exported to", dasfile.Name())
fmt.Println(len(devsIpv4sGraphs), "devices exported to", dasfile.Name())
// Test some well-known nodes and devices
// node_id := []int{53475, 53537}
......@@ -172,17 +211,17 @@ func main() {
// Remove from AntidoteDB those devices that are not in the CNML file anymore
for k, v := range antidoteDevices {
if !isDeviceIPv4AddressesInArray(v, devsadds) {
if !isDeviceIpv4sGraphserverInArray(v, devsIpv4sGraphs) {
// Remove IPv4 addresses of the removed device
for _, w := range readDeviceIpv4AddressesFromAntidote(v).Addresses {
if removeIpv4addressFromDevice(v, w) {
for _, w := range readDeviceIpv4sGraphserverFromAntidote(v).Addresses {
if removeIpv4addressFromDeviceIpv4sGraphserver(v, w) {
antidoteDeleteAddressSuccess++
} else {
antidoteDeleteAddressFail++
}
}
// Remove the device from the list
if removeDeviceFromAntidote(v) {
if removeDeviceFromAntidote(v.ID) {
antidoteDeleteDeviceSuccess++
} else {
antidoteDeleteDeviceFail++
......@@ -190,10 +229,10 @@ func main() {
// If the device is in the CNML file, check for old IPv4 addresses no longer in use and remove them
} else {
for _, w := range devsadds {
for _, w := range devsIpv4sGraphs {
for _, x := range v.Addresses {
if !isIPv4addressInDevicesArray(w, x) {
if removeIpv4addressFromDevice(w, x) {
if !isIPv4addressInDeviceIpv4sGraphserversArray(w, x) {
if removeIpv4addressFromDeviceIpv4sGraphserver(w, x) {
antidoteDeleteAddressSuccess++
} else {
antidoteDeleteAddressFail++
......@@ -216,12 +255,12 @@ func main() {
fmt.Printf("\r%d IPv4 addresses removed from AntidoteDB (%d success, %d fail)\n", antidoteDeleteAddressSuccess+antidoteDeleteAddressFail, antidoteDeleteAddressSuccess, antidoteDeleteAddressFail)
// Add to AntidoteDB those devices that are new in the CNML file
for k, v := range devsadds {
for k, v := range devsIpv4sGraphs {
// Check if the device is in Antidote already and add it if missing
saveDevFail := false
if !isDeviceIPv4AddressesInArray(v, antidoteDevices) && len(v.Addresses) > 0 {
if !isDeviceIpv4sGraphserverInArray(v, antidoteDevices) && len(v.Addresses) > 0 {
// Add the device to the devices set
if saveDeviceToAntidote(v) {
if saveDeviceIpv4sGraphserverToAntidote(v) {
antidoteAddDeviceSuccess++
} else {
antidoteAddDeviceFail++
......@@ -232,7 +271,7 @@ func main() {
if !saveDevFail {
// Iterate through all the device's addresses and add them if new
for _, w := range v.Addresses {
if saveIpv4addresToDeviceToAntidote(v, w) {
if saveIpv4addresToDeviceIpv4sGraphserverToAntidote(v, w) {
antidoteAddAddressSuccess++
} else {
antidoteAddAddressFail++
......@@ -462,8 +501,8 @@ func getZonesInZoneExc(zone cnml.Zone) []cnml.Zone {
}
// Remove a device by its ID from Antidote using João's REST server
func removeDeviceFromAntidote(device cnml.DeviceIpv4Adddresses) bool {
return apidote.AntidoteRemoveItemFromSetInBucket("guifi", "devices", fmt.Sprintf("%d", device.ID))
func removeDeviceFromAntidote(ID int) bool {
return apidote.AntidoteRemoveItemFromSetInBucket("guifi", "devices", strconv.Itoa(ID))
}
// Read all the devices from Antidote using João's REST server
......@@ -476,17 +515,41 @@ func readDeviceIpv4AddressesFromAntidote(device cnml.DeviceIpv4Adddresses) cnml.
return device
}
// Read all the devices from Antidote using João's REST server
func readDeviceIpv4sGraphserverFromAntidote(device cnml.DeviceIpv4sGraphserver) cnml.DeviceIpv4sGraphserver {
for _, v := range apidote.AntidoteReadItemsFromSetInBucket("guifi", fmt.Sprintf("device-%d", device.ID)) {
device.Addresses = append(device.Addresses, v)
}
return device
}
// Save a device ID to Antidote using João's REST server
func saveDeviceToAntidote(device cnml.DeviceIpv4Adddresses) bool {
return apidote.AntidoteAddItemToSetInBucket("guifi", "devices", fmt.Sprintf("%d", device.ID))
}
// Save a device ID to Antidote using João's REST server
func saveDeviceIpv4sGraphserverToAntidote(device cnml.DeviceIpv4sGraphserver) bool {
return apidote.AntidoteAddItemToSetInBucket("guifi", "devices", fmt.Sprintf("%d", device.ID))
}
// Save an IPv4 address to a device
func saveIpv4addresToDeviceToAntidote(device cnml.DeviceIpv4Adddresses, address string) bool {
return apidote.AntidoteAddItemToSetInBucket("guifi", fmt.Sprintf("device-%d", device.ID), address)
}
func removeIpv4addressFromDevice(device cnml.DeviceIpv4Adddresses, address string) bool {
// Save an IPv4 address to a device
func saveIpv4addresToDeviceIpv4sGraphserverToAntidote(device cnml.DeviceIpv4sGraphserver, address string) bool {
return apidote.AntidoteAddItemToSetInBucket("guifi", fmt.Sprintf("device-%d", device.ID), address)
}
func removeIpv4addressFromDeviceIpv4Adddresses(device cnml.DeviceIpv4Adddresses, address string) bool {
return apidote.AntidoteRemoveItemFromSetInBucket("guifi", fmt.Sprintf("device-%d", device.ID), address)
}
func removeIpv4addressFromDeviceIpv4sGraphserver(device cnml.DeviceIpv4sGraphserver, address string) bool {
return apidote.AntidoteRemoveItemFromSetInBucket("guifi", fmt.Sprintf("device-%d", device.ID), address)
}
......@@ -500,6 +563,16 @@ func isDeviceIPv4AddressesInArray(device cnml.DeviceIpv4Adddresses, devices []cn
return false
}
// Check if an int item is in an ints array
func isDeviceIpv4sGraphserverInArray(device cnml.DeviceIpv4sGraphserver, devices []cnml.DeviceIpv4sGraphserver) bool {
for _, v := range devices {
if v.ID == device.ID {
return true
}
}
return false
}
func isIPv4addressInDevicesArray(device cnml.DeviceIpv4Adddresses, address string) bool {
for _, v := range device.Addresses {
if v == address {
......@@ -509,9 +582,19 @@ func isIPv4addressInDevicesArray(device cnml.DeviceIpv4Adddresses, address strin
return false
}
func isIPv4addressInDeviceIpv4sGraphserversArray(device cnml.DeviceIpv4sGraphserver, address string) bool {
for _, v := range device.Addresses {
if v == address {
return true
}
}
return false
}
// Panic on error
func errCheck(e error) {
if e != nil {
panic(e)
func errCheck(err error, message string) {
if err != nil {
fmt.Println(message)
panic(err)
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment