Most of these examples come from the FreeCONF examples git repo.
This is the multi-page printable view of this section. Click here to print.
Examples
Every type of example imaginable and a few not imaginable.
- 1: Next Step
- 2: Development Guide
- 3: RESTCONF client
- 4: gNMI server
- 5: gNMIc
- 6: Ansible
- 7: InfluxDB
- 8: Prometheus
- 9: Slack
- 10: Node/Reflect
- 11: Node/Extend
- 12: Node/anydata
- 13: Code generation
- 14: Debugging
- 15: Unit Testing
- 16: Web UIs
- 17: Request filters
1 - Next Step
After “Getting Started”
This document goes past Getting Started by exploring YANG’s fundamental management types:
- configuration
- metrics
- RPCs
- notifications
We’ll demonstrate thru example by writing software that represents a car. This car has configuration such as speed, metrics such as the odometer, rpcs such as start and stopping the car and events such as getting a flat tire.
You can structure your code however you like, but here we are breaking this example into five separate files:
- Car main source
- Car unit test
- YANG to describe management interface
- Management source using FreeCONF
- Management unit test
1. Car main source
This is just normal code to model the car. Notice there are no references to FreeCONF library, this is just code as you might normally write it.
file : car.go
package car
import (
"container/list"
"fmt"
"math/rand"
"time"
)
// ////////////////////////
// C A R - example application
//
// This has nothing to do with FreeCONF, just an example application written in Go.
// that models a running car that can get flat tires when tires are worn.
type Car struct {
Tire []*Tire
Miles float64
Running bool
// When the tires were last rotated
LastRotation int64
// Default speed value is in yang model file and free's your code
// from hardcoded values, even if they are only default values
// units milliseconds/mile
Speed int
// How fast to apply speed. Default it 1s or "miles per second"
PollInterval time.Duration
// Listeners are common on manageable code. Having said that, listeners
// remain relevant to your application. The manage.go file is responsible
// for bridging the conversion from application to management api.
listeners *list.List
}
// CarListener for receiving car update events
type CarListener func(UpdateEvent)
// car event types
type UpdateEvent int
const (
CarStarted UpdateEvent = iota + 1
CarStopped
FlatTire
)
func (e UpdateEvent) String() string {
strs := []string{
"unknown",
"carStarted",
"carStopped",
"flatTire",
}
if int(e) < len(strs) {
return strs[e]
}
return "invalid"
}
func New() *Car {
c := &Car{
listeners: list.New(),
Speed: 1000,
PollInterval: time.Second,
}
c.NewTires()
return c
}
// Stop will take up to poll_interval seconds to come to a stop
func (c *Car) Stop() {
c.Running = false
}
func (c *Car) Start() {
if c.Running {
return
}
go func() {
c.Running = true
c.updateListeners(CarStarted)
defer func() {
c.Running = false
c.updateListeners(CarStopped)
}()
for c.Speed > 0 {
poll := time.NewTicker(c.PollInterval)
for c.Running {
for range poll.C {
c.Miles += float64(c.Speed)
for _, t := range c.Tire {
t.endureMileage(c.Speed)
if t.Flat {
c.updateListeners(FlatTire)
return
}
}
}
}
}
}()
}
// OnUpdate to listen for car events like start, stop and flat tire
func (c *Car) OnUpdate(l CarListener) Subscription {
return NewSubscription(c.listeners, c.listeners.PushBack(l))
}
func (c *Car) NewTires() {
c.Tire = make([]*Tire, 4)
c.LastRotation = int64(c.Miles)
for pos := 0; pos < len(c.Tire); pos++ {
c.Tire[pos] = &Tire{
Pos: pos,
Wear: 100,
Size: "H15",
}
}
}
func (c *Car) ReplaceTires() {
for _, t := range c.Tire {
t.Replace()
}
c.LastRotation = int64(c.Miles)
}
func (c *Car) RotateTires() {
x := c.Tire[0]
c.Tire[0] = c.Tire[1]
c.Tire[1] = c.Tire[2]
c.Tire[2] = c.Tire[3]
c.Tire[3] = x
for i, t := range c.Tire {
t.Pos = i
}
c.LastRotation = int64(c.Miles)
}
func (c *Car) updateListeners(e UpdateEvent) {
fmt.Printf("car %s\n", e)
i := c.listeners.Front()
for i != nil {
i.Value.(CarListener)(e)
i = i.Next()
}
}
// T I R E
type Tire struct {
Pos int
Size string
Flat bool
Wear float64
Worn bool
}
func (t *Tire) Replace() {
t.Wear = 100
t.Flat = false
t.Worn = false
}
func (t *Tire) checkIfFlat() {
if !t.Flat {
// emulate that the more wear a tire has, the more likely it will
// get a flat, but there is always a chance.
t.Flat = (t.Wear - (rand.Float64() * 10)) < 0
}
}
func (t *Tire) endureMileage(speed int) {
// Wear down [0.0 - 0.5] of each tire proportionally to the tire position
t.Wear -= (float64(speed) / 100) * float64(t.Pos) * rand.Float64()
t.checkIfFlat()
t.checkForWear()
}
func (t *Tire) checkForWear() bool {
return t.Wear < 20
}
///////////////////////
// U T I L
// Subscription is handle into a list.List that when closed
// will automatically remove item from list. Useful for maintaining
// a set of listeners that can easily remove themselves.
type Subscription interface {
Close() error
}
// NewSubscription is used by subscription managers to give a token
// to caller the can close to unsubscribe to events
func NewSubscription(l *list.List, e *list.Element) Subscription {
return &listSubscription{l, e}
}
type listSubscription struct {
l *list.List
e *list.Element
}
// Close will unsubscribe to events.
func (sub *listSubscription) Close() error {
sub.l.Remove(sub.e)
return nil
}
file : car.py
import freeconf.nodeutil
import time
import threading
import random
# If these strings match YANG enum ids then they will be converted automatically. Ints would
# work as well
EVENT_STARTED = "carStarted"
EVENT_STOPPED = "carStopped"
EVENT_FLAT_TIRE = "flatTire"
# Simple application, no connection to management
# C A R
# Your application code.
#
# Notice there are no reference to FreeCONF in this file. This means your
# code remains:
# - unit test-able
# - Not auto-generated from model files
# - free of annotations/tags
class Car():
def __init__(self):
# metrics/state
self.miles = 0
self.running = False
self.thread = None
# potential configuable fields
self.speed = 1000
self.new_tires()
self.poll_interval = 1.0 #secs
# Listeners are common on manageable code. Having said that, listeners
# remain relevant to your application. The manage.go file is responsible
# for bridging the conversion from application to management api.
self.listeners = []
def start(self):
if self.running:
return
self.thread = threading.Thread(target=self.run, name="Car")
self.thread.start()
def stop(self):
"""
Will take up to poll_interval seconds to come to a stop
"""
self.running = False
def run(self):
try:
self.running = True
self.update_listeners(EVENT_STARTED)
while self.running:
time.sleep(self.poll_interval)
self.miles = self.miles + self.speed
for t in self.tire:
t.endure_mileage(self.speed)
if t.flat:
self.update_listeners(EVENT_FLAT_TIRE)
return
finally:
self.running = False
self.update_listeners(EVENT_STOPPED)
def on_update(self, listener):
"""
listen for car events like start, stop and flat tire
"""
self.listeners.append(listener)
def closer():
self.listeners.remove(listener)
return closer
def update_listeners(self, event):
print(f"car {event}")
for l in self.listeners:
l(event)
def new_tires(self):
self.tire = []
for pos in range(4):
self.tire.append(Tire(pos))
self.last_rotation = self.miles
def replace_tires(self):
for t in self.tire:
t.replace()
self.last_rotation = int(self.miles)
self.start()
def rotate_tires(self):
first = self.tire[0]
self.tire[0] = self.tire[1]
self.tire[1] = self.tire[2]
self.tire[2] = self.tire[3]
self.tire[3] = first
for i in range(len(self.tire)):
self.tire[i].pos = i
self.last_rotation = int(self.miles)
class Tire:
def __init__(self, pos):
self.pos = pos
self.wear = 100
self.size = "H15"
self.flat = False
self.worn = False
def replace(self):
self.wear = 100
self.flat = False
self.worn = False
def check_if_flat(self):
if not self.flat:
# emulate that the more wear a tire has, the more likely it will
# get a flat, but there is always a chance.
self.flat = (self.wear - (random.random() * 10)) < 0
def endure_mileage(self, speed):
# Wear down [0.0 - 0.5] of each tire proportionally to the tire position
self.wear -= (float(speed) / 100) * float(self.pos) * random.random()
self.check_if_flat()
self.check_for_wear()
def check_for_wear(self):
self.wear < 20
2. Car unit test
Shown here for completeness.
file : car_test.go
package car
import (
"fmt"
"testing"
"time"
"github.com/freeconf/yang/fc"
)
// Quick test of car's features using direct access to fields and methods
// again, nothing to do with FreeCONF.
func TestCar(t *testing.T) {
c := New()
c.PollInterval = time.Millisecond
c.Speed = 1000
events := make(chan UpdateEvent)
unsub := c.OnUpdate(func(e UpdateEvent) {
fmt.Printf("got event %s\n", e)
events <- e
})
t.Log("waiting for car events...")
c.Start()
fc.AssertEqual(t, CarStarted, <-events)
fc.AssertEqual(t, FlatTire, <-events)
fc.AssertEqual(t, CarStopped, <-events)
c.ReplaceTires()
c.Start()
fc.AssertEqual(t, CarStarted, <-events)
unsub.Close()
c.Stop()
}
file : test_car.py
#!/usr/bin/env python3
import unittest
import queue
import car
class TestCar(unittest.TestCase):
# Quick test of car's features using direct access to fields and methods
def test_server(self):
c = car.Car()
c.poll_interval = 0.01
c.speed = 1000
events = queue.Queue()
def on_update(e):
print(f"got event {e}")
events.put(e)
unsub = c.on_update(on_update)
print("waiting for car events...")
c.start()
self.assertEqual(car.EVENT_STARTED, events.get())
self.assertEqual(car.EVENT_FLAT_TIRE, events.get())
self.assertEqual(car.EVENT_STOPPED, events.get())
c.replace_tires()
c.start()
self.assertEqual(car.EVENT_STARTED, events.get())
unsub()
c.replace_tires()
c.stop()
if __name__ == '__main__':
unittest.main()
3. YANG to describe management interface
Come up with YANG file for the desired management API for your car. If you need help understanding YANG files there are a lot of references on the internet including this YANG primer.
file : car.yang
// Every yang file has a single module (or sub-module) definition. The name of the module
// must match the name of the file. So module definition for "car" would be in "car.yang".
// Only exception to this rule is advanced naming schemes that introduce version into
// file name.
module car {
description "Car goes beep beep";
revision 2023-03-27; // date YYYY-MM-DD is typical but you can use any scheme
// globally unique id when used with module name should this YANG file mingles with other systems
namespace "freeconf.org";
prefix "car"; // used when importing definitions from other files which we don't use here
// While any order is fine, it does control the order of data returned in the management
// interface or the order configuration is applied. You can mix order of metrics and
// config, children, rpcs, notifications as you see fit
// begin car root config...
leaf speed {
description "how many miles the car travels in one poll interval";
type int32;
units milesPerSecond;
default 1000;
}
leaf pollInterval {
description "time between traveling ${speed} miles";
type int32;
units millisecs;
default 1000;
}
// begin car root metrics...
leaf running {
description "state of the car moving or not";
type boolean;
config false;
}
leaf miles {
description "odometer - how many miles has car moved";
config false;
type decimal64 {
fraction-digits 2;
}
}
leaf lastRotation {
description "the odometer reading of the last tire rotation";
type int64;
config false;
}
// begin children objects of car...
list tire {
description "rubber circular part that makes contact with road";
// lists are most helpful when you identify a field or fields that uniquely identifies
// the items in the list. This is not strictly neccessary.
key pos;
leaf pos {
description "numerical positions of 0 thru 3";
type int32;
}
// begin tire config...
leaf size {
description "informational information of the size of the tire";
type string;
default "H15";
}
// begin tire metrics
leaf worn {
description "a somewhat subjective but deterministic value of the amount of
wear on a tire indicating tire should be replaced soon";
config false;
type boolean;
}
leaf wear {
description "number representing the amount of wear and tear on the tire.
The more wear on a tire the more likely it is to go flat.";
config false;
type decimal64 {
fraction-digits 2;
}
}
leaf flat {
description "tire has a flat and car would be kept from running. Use
replace tire or tires to get car running again";
config false;
type boolean;
}
// begin tire RPCs...
action replace {
description "replace just this tire";
// simple rpc with no input or output.
// Side note: you could have designed this as an rpc at the root level that
// accepts tire position as a single argument but putting it here makes it
// more natural and simple to call.
}
}
// In YANG 'rpc' and 'action' are identical but for historical reasons you must only
// use 'rpc' only when on the root and 'action' when inside a container or list.
// begin car RPCs...
rpc reset {
description "reset the odometer"; // somewhat unrealistic of a real car odometer
}
rpc rotateTires {
description "rotate tires for optimal wear";
}
rpc replaceTires {
description "replace all tires with fresh tires and no wear";
}
rpc start {
description "start the car if it is not already started";
}
rpc stop {
description "stop the car if it is not already stopped";
}
// begin of car events...
notification update {
description "important state information about your car";
leaf event {
type enumeration {
enum carStarted {
// optional. by default the values start at 0 and increment 1 past the
// previous value. Numbered values may be even ever be used in your programs
// and therefore irrelevant. Here I define a value just to demonstrate I could.
value 1;
}
enum carStopped;
enum flatTire;
}
}
}
}
4. Management source using FreeCONF
Here we use FreeCONF’s nodeutil.Node that uses reflection to implement the managment functions.
This is far from the only way to implement this, you can generate code or implement your own logic that implements the node.Node interface.
file : manage.go
package car
import (
"reflect"
"time"
"github.com/freeconf/yang/meta"
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
)
// ///////////////////////
// C A R M A N A G E M E N T
//
// Manage your car application using FreeCONF library according to the car.yang
// model file.
//
// Manage is root handler from car.yang. i.e. module car { ... }
func Manage(car *Car) node.Node {
// We're letting reflect do a lot of the work when the yang file matches
// the field names and methods in the objects. But we extend reflection
// to add as much custom behavior as we want
return &nodeutil.Node{
// Initial object. Note: as the tree is traversed, new Node instances
// will have different values in their Object reference
Object: car,
// implement RPCs
OnAction: func(n *nodeutil.Node, r node.ActionRequest) (node.Node, error) {
switch r.Meta.Ident() {
case "reset":
// here we implement custom handler for action just as an example
// If there was a Reset() method on car the default OnAction
// handler would be able to call Reset like all the other functions
car.Miles = 0
default:
// all the actions like start, stop and rotateTire are called
// thru reflecton here because their method names align with
// the YANG.
return n.DoAction(r)
}
return nil, nil
},
// implement yang notifications (which are really just event streams)
OnNotify: func(p *nodeutil.Node, r node.NotifyRequest) (node.NotifyCloser, error) {
switch r.Meta.Ident() {
case "update":
// can use an adhoc struct send event
sub := car.OnUpdate(func(e UpdateEvent) {
msg := struct {
Event int
}{
Event: int(e),
}
// events are nodes too
r.Send(nodeutil.ReflectChild(&msg))
})
// we return a close **function**, we are not actually closing here
return sub.Close, nil
}
// there is no default implementation at this time, all notification streams
// require custom handlers.
return p.Notify(r)
},
// implement fields that are not automatically handled by reflection.
OnRead: func(p *nodeutil.Node, r meta.Definition, t reflect.Type, v reflect.Value) (reflect.Value, error) {
if l, ok := r.(meta.Leafable); ok {
// use "units" in yang to tell what to convert.
//
// Other useful ways to intercept custom reflection reads:
// 1.) incoming reflect.Type t
// 2.) field name used in yang or some pattern of the name (suffix, prefix, regex)
// 3.) yang extension
// 4.) any combination of these
if l.Units() == "millisecs" {
return reflect.ValueOf(v.Int() / int64(time.Millisecond)), nil
}
}
return v, nil
},
// Generally the reverse of what is handled in OnRead
OnWrite: func(p *nodeutil.Node, r meta.Definition, t reflect.Type, v reflect.Value) (reflect.Value, error) {
if l, ok := r.(meta.Leafable); ok {
if l.Units() == "millisecs" {
d := time.Duration(v.Int()) * time.Millisecond
return reflect.ValueOf(d), nil
}
}
return v, nil
},
}
}
file : manage.py
from freeconf import nodeutil, val
#
# C A R M A N A G E M E N T
# Bridge from model to car app.
#
# manage is root handler from car.yang. i.e. module car { ... }
def manage(c):
# implement RPCs (action or rpc)
def action(n, req):
if req.meta.ident == 'reset':
c.miles = 0
return None
return n.do_action(req)
# implement yang notifications which are really just events
def notify(n, r):
if r.meta.ident == 'update':
def listener(event):
# events are nodes too
r.send(nodeutil.Node(object={
"event": event
}))
closer = c.on_update(listener)
return closer
return n.do_notify(r)
# implement fields that are not automatically handled by reflection.
def read(_n, meta, v):
if meta.units == 'millisecs':
return int(v * 1000)
return v
def write(_n, meta, v):
if meta.units == 'millisecs':
return float(v) / 1000 # ms to secs
return v
# Extend and Reflect form a powerful combination, we're letting reflect do a lot of the CRUD
# when the yang file matches the field names. But we extend reflection
# to add as much custom behavior as we want
return nodeutil.Node(c,
on_action = action, on_notify=notify, on_read=read, on_write=write)
5. Management unit test
Write unit test for you management API without starting a server. You can read or write configuration, call functions, listen to events or read metrics all from your unit test.
file : manage_test.go
package car
import (
"testing"
"github.com/freeconf/yang/fc"
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
"github.com/freeconf/yang/parser"
"github.com/freeconf/yang/source"
)
// Test the car management logic in manage.go
func TestManage(t *testing.T) {
// setup
ypath := source.Path("../yang")
mod := parser.RequireModule(ypath, "car")
app := New()
// no web server needed, just your app and management function.
brwsr := node.NewBrowser(mod, Manage(app))
root := brwsr.Root()
// read all config
currCfg, err := nodeutil.WriteJSON(sel(root.Find("?content=config")))
fc.AssertEqual(t, nil, err)
expected := `{"speed":1000,"pollInterval":1000,"tire":[{"pos":0,"size":"H15"},{"pos":1,"size":"H15"},{"pos":2,"size":"H15"},{"pos":3,"size":"H15"}]}`
fc.AssertEqual(t, expected, currCfg)
// access car and verify w/API
fc.AssertEqual(t, false, app.Running)
// setup event listener, verify events later
events := make(chan string)
unsub, err := sel(root.Find("update")).Notifications(func(n node.Notification) {
event, _ := nodeutil.WriteJSON(n.Event)
events <- event
})
fc.AssertEqual(t, nil, err)
fc.AssertEqual(t, 1, app.listeners.Len())
// write config starts car
n, err := nodeutil.ReadJSON(`{"speed":2000}`)
fc.AssertEqual(t, nil, err)
err = root.UpdateFrom(n)
fc.AssertEqual(t, nil, err)
fc.AssertEqual(t, 2000, app.Speed)
// start car
fc.AssertEqual(t, nil, justErr(sel(root.Find("start")).Action(nil)))
// should be first event
fc.AssertEqual(t, `{"event":"carStarted"}`, <-events)
fc.AssertEqual(t, true, app.Running)
// unsubscribe
unsub()
fc.AssertEqual(t, 0, app.listeners.Len())
// hit all the RPCs
fc.AssertEqual(t, nil, justErr(sel(root.Find("rotateTires")).Action(nil)))
fc.AssertEqual(t, nil, justErr(sel(root.Find("replaceTires")).Action(nil)))
fc.AssertEqual(t, nil, justErr(sel(root.Find("reset")).Action(nil)))
fc.AssertEqual(t, nil, justErr(sel(root.Find("tire=0/replace")).Action(nil)))
}
func sel(s *node.Selection, err error) *node.Selection {
if err != nil {
panic(err)
}
return s
}
func justErr(_ *node.Selection, err error) error {
return err
}
file : test_manage.py
#!/usr/bin/env python3
import unittest
from freeconf import parser, node, source, nodeutil
import manage
import queue
import car
import time
# Test the car management API in manage.py
class TestManage(unittest.TestCase):
def test_manage(self):
# where YANG files are
ypath = source.path("../yang")
mod = parser.load_module_file(ypath, 'car')
# create your empty app instance
app = car.Car()
# no web server needed, just your app instance and management entry point.
brwsr = node.Browser(mod, manage.manage(app))
# get a selection into the management API to begin interaction
root = brwsr.root()
# TEST CONFIG GET: read all the config
curr_cfg = nodeutil.json_write_str(root.find("?content=config"))
expected = '{"speed":1000,"pollInterval":1000,"tire":[{"pos":0,"size":"H15"},{"pos":1,"size":"H15"},{"pos":2,"size":"H15"},{"pos":3,"size":"H15"}]}'
self.assertEqual(expected, curr_cfg)
# verify car starts not running. Here we are checking from the car instance itsel
self.assertEqual(False, app.running)
# SETUP EVENT TEST: here we add a listener to receive events and stick them in a thread-safe
# queue for assertion checking later
events = queue.Queue()
def on_update(n):
msg = nodeutil.json_write_str(n.event)
events.put(msg)
unsub = root.find("update").notification(on_update)
# TEST CONFIG SET: write config
root.update_from(nodeutil.json_read_str('{"speed":2000}'))
self.assertEqual(2000, app.speed)
# TEST RPC: start car
root.find("start").action()
# TEST EVENT: verify first event. will block until event comes in
print("waiting for event...")
self.assertEqual('{"event":"carStarted"}', events.get())
print("event receieved.")
# TEST EVENT UNSUBSCRIBE: done listening for events
unsub()
# TEST RPCS: just hit all the RPCs for code coverage. You could easily add the
# underlying car object is changed accordingly
root.find("rotateTires").action()
root.find("replaceTires").action()
root.find("reset").action()
self.assertEqual(0, app.miles)
root.find("tire=0/replace").action()
print("done")
if __name__ == '__main__':
unittest.main()
What’s Next?
- YANG primer - to learn more about building your YANG
- Code generation - To assist with reducing the code to write and improving the compile-time safety between code and YANG file
- More examples for building your management interface using Basic, Extends and Reflect
2 - Development Guide
Building Go nodes by using abstract class
Use cases:
- high-level routing areas
listnodes- areas with not a lot of CRUD
- bridges to systems that are not Go structs (e.g. DB, YAML, external REST APIs, etc.)
- part of code generation
Highlevel routing
Code
package demo
type App struct {
users *UserService
fonts *FontManager
bagels *BagelMaker
}
func NewApp() *App {
return &App{
users: &UserService{},
fonts: &FontManager{},
bagels: &BagelMaker{},
}
}
type UserService struct{}
type FontManager struct{}
type BagelMaker struct{}
class App:
def __init__(self):
self.users = {}
self.fonts = {}
self.bagels = {}
YANG
module my-app {
namespace "freeconf.org";
prefix "a";
container users {
// ... more stuff here
}
container fonts {
// ... more stuff here
}
container bagels {
// ... more stuff here
}
}
…then your node code can be this.
package demo
import (
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
)
func manage(a *App) node.Node {
return &nodeutil.Basic{
OnChild: func(r node.ChildRequest) (node.Node, error) {
switch r.Meta.Ident() {
case "users":
return nodeutil.ReflectChild(a.users), nil
case "fonts":
return nodeutil.ReflectChild(a.fonts), nil
case "bagels":
return nodeutil.ReflectChild(a.bagels), nil
}
return nil, nil
},
}
}
You cannot use Reflect here because fields are private.
from freeconf import nodeutil
def manage_app(app):
def child(req):
if req.meta.ident == 'users':
return nodeutil.Node(app.users)
elif req.meta.ident == 'fonts':
return nodeutil.Node(app.fonts)
elif req.meta.ident == 'bagels':
return nodeutil.Node(app.bagels)
return None
# while this could easily be nodeutil.Node, we illustrate a Basic
# node should you want essentially an abstract class that stubs all
# this calls with reasonable default handlers
return nodeutil.Basic(on_child=child)
Additional Files
file: manage_test.go
package demo
import (
"testing"
"github.com/freeconf/yang/fc"
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
"github.com/freeconf/yang/parser"
"github.com/freeconf/yang/source"
)
func TestManage(t *testing.T) {
a := NewApp()
ypath := source.Dir(".")
m := parser.RequireModule(ypath, "my-app")
bwsr := node.NewBrowser(m, manage(a))
root := bwsr.Root()
defer root.Release()
actual, err := nodeutil.WriteJSON(root)
fc.AssertEqual(t, nil, err)
fc.AssertEqual(t, `{"users":{},"fonts":{},"bagels":{}}`, actual)
}
file: test_manage.py
#!/usr/bin/env python3
import unittest
from freeconf import parser, nodeutil, node, source
from manage import manage_app
from app import App
class TestManage(unittest.TestCase):
def test_manage(self):
app = App()
ypath = source.path("..")
m = parser.load_module_file(ypath, 'my-app')
mgmt = manage_app(app)
bwsr = node.Browser(m, mgmt)
root = bwsr.root()
try:
actual = nodeutil.json_write_str(root)
self.assertEqual('{"users":{},"fonts":{},"bagels":{}}', actual)
finally:
root.release()
if __name__ == '__main__':
unittest.main()
3 - RESTCONF client
Connecting to a RESTCONF server
You can communicate with a RESTCONF server using a basic HTTP library. For simple applications this is a reasonable approach. If you want to “level-up” your API access to be able to walk the schema (i.e. YANG) then you can use the RESTCONF client in FreeCONF.
package demo
import (
"fmt"
"strings"
"testing"
"github.com/freeconf/examples/car"
"github.com/freeconf/restconf"
"github.com/freeconf/restconf/client"
"github.com/freeconf/restconf/device"
"github.com/freeconf/yang/fc"
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
"github.com/freeconf/yang/source"
)
func connectClient() {
// YANG: just need YANG file ietf-yang-library.yang, not the yang of remote system as that will
// be downloaded as needed
ypath := restconf.InternalYPath
// Connect
proto := client.ProtocolHandler(ypath)
dev, err := proto("http://localhost:9998/restconf")
if err != nil {
panic(err)
}
// Get a browser to walk server's management API for car
car, err := dev.Browser("car")
if err != nil {
panic(err)
}
root := car.Root()
defer root.Release()
// Example of config: I feel the need, the need for speed
// bad config is rejected in client before it is sent to server
n, err := nodeutil.ReadJSON(`{"speed":100}`)
if err != nil {
panic(err)
}
err = root.UpsertFrom(n)
if err != nil {
panic(err)
}
// Example of metrics: Get all metrics as JSON
sel, err := root.Find("?content=nonconfig")
if err != nil {
panic(err)
}
defer sel.Release()
metrics, err := nodeutil.WriteJSON(sel)
if err != nil {
panic(err)
}
if metrics == "" {
panic("no metrics")
}
// Example of RPC: Reset odometer
sel, err = root.Find("reset")
if err != nil {
panic(err)
}
defer sel.Release()
if _, err = sel.Action(nil); err != nil {
panic(err)
}
// Example of notification: Car has an important update
sel, err = root.Find("update")
if err != nil {
panic(err)
}
defer sel.Release()
unsub, err := sel.Notifications(func(n node.Notification) {
msg, err := nodeutil.WriteJSON(n.Event)
if err != nil {
panic(err)
}
fmt.Println(msg)
})
if err != nil {
panic(err)
}
defer unsub()
// Example of multiple modules: This is the FreeCONF server module
rcServer, err := dev.Browser("fc-restconf")
if err != nil {
panic(err)
}
// Example of config: Enable debug logging on FreeCONF's remote RESTCONF server
serverSel := rcServer.Root()
defer serverSel.Release()
n, err = nodeutil.ReadJSON(`{"debug":true}`)
if err != nil {
panic(err)
}
serverSel.UpsertFrom(n)
if err != nil {
panic(err)
}
}
func TestClient(t *testing.T) {
// setup - start a server
ypath := source.Path("../yang")
serverYPath := source.Any(ypath, restconf.InternalYPath)
carServer := car.New()
local := device.New(serverYPath)
local.Add("car", car.Manage(carServer))
s := restconf.NewServer(local)
defer s.Close()
cfg := `{
"fc-restconf": {
"debug": true,
"web" : {
"port": ":9998"
}
},
"car" : {
"speed": 5
}
}`
fc.RequireEqual(t, nil, local.ApplyStartupConfig(strings.NewReader(cfg)))
connectClient()
}
4 - gNMI server
Adding gNMI server-side support to your application
gNMI is a alternative communication protocol to RESTCONF by openconfig.net](https://www.openconfig.net). You might use gNMI over RESTCONF because the services you want to use work with gNMI. With FreeCONF you can easily enable RESTCONF, gNMI or both at the same time.
How to add gNMI server support to your application
file: main.go
package main
import (
"flag"
"log"
"github.com/freeconf/examples/car"
"github.com/freeconf/gnmi"
"github.com/freeconf/restconf/device"
"github.com/freeconf/yang/source"
)
// Connect everything together into a server to start up
func main() {
flag.Parse()
// Your app here
app := car.New()
// where the yang files are stored
ypath := source.Any(gnmi.InternalYPath, source.Path("../yang"))
// Device is just a container for browsers. Needs to know where YANG files are stored
d := device.New(ypath)
// Device can hold multiple modules, here we are only adding one
if err := d.Add("car", car.Manage(app)); err != nil {
panic(err)
}
// Select wire-protocol gNMI to serve the device
gnmi.NewServer(d)
// apply start-up config normally stored in a config file on disk
if err := d.ApplyStartupConfigFile("./startup.json"); err != nil {
panic(err)
}
if !*testMode {
// wait for ctrl-c
log.Printf("server started")
select {}
}
}
var testMode = flag.Bool("test", false, "do not run in background (i.e. driven by unit test)")
To run this example
git clone https://github.com/freeconf/examples fc-examples
cd ./gnmi-server
go run .
Checkout FreeCONF gNMIc examples for interacting with this running service.
To add gNMI support to your Go application
Just add the Go dependency and setup for gnmi.NewServer where ever makes sense to your application.
go get github.com/freeconf/gnmi
5 - gNMIc
Using the gNMIc client to connect to your application
gNMIc is a utility that can connect to any appliction that supports gNMI protocol including applications. See how to add gNMI support to your application using the FreeCONF. gnmic is unique because you can use it from the terminal to diagnose issues or use it in production to send metrics to InfluxDB, Prometheus, Kafka, NATS.
Here are some working examples of using gnmic from terminal to connect to the car example application with gnmi server support added.
Setup
Downloading FreeCONF example source code
git clone https://github.com/freeconf/examples fc-examples
Setup gnmi server
Run the example gNMI server with car application registered with server
cd ./gnmi-server
go run .
Running gnmic
First Install the gNMIc client executable. Then, in a separate terminal, go to the directory containing gnmic commands.
cd openconfig-gnmi
Example commands
#!/usr/bin/env bash
set -euf -o pipefail
# Get Data Object
gnmic --config car.yml get --path car:/
# Get Data Value
gnmic --config car.yml get --path car:/speed
# Get Data : use model.
gnmic --config car.yml get --model car --path /
# Set Data Object
gnmic --config car.yml set --update car:/:::json:::'{"speed":300}'
# Set Data Value
gnmic --config car.yml set --update-path car:/speed --update-value 300
# Subscribe
timeout 5s \
gnmic --config car.yml sub --model car --path / --sample-interval 1s --heartbeat-interval 2s || true
# Subscribe to just tire metrics : use model
timeout 5s \
gnmic --config car.yml sub --mode once --model car --path /tire || true
# Subscribe to just tire metrics
timeout 5s \
gnmic --config car.yml sub --mode once --path car:/tire || true
# Subscribe to just tire 1 metrics
timeout 5s \
gnmic --config car.yml sub --mode once --path car:/tire=1 || true
Example Output
Command: gnmic --config car.yml --model car --path /
[
{
"source": "localhost:8090",
"timestamp": 1680564190640829839,
"time": "2023-04-03T19:23:10.640829839-04:00",
"updates": [
{
"Path": "",
"values": {
"": {
"lastRotation": 0,
"miles": 4100,
"running": true,
"speed": 10,
"tire": [
{
"flat": false,
"pos": 0,
"size": "15",
"wear": 100,
"worn": false
},
{
"flat": false,
"pos": 1,
"size": "15",
"wear": 79.28663660297573,
"worn": false
},
{
"flat": false,
"pos": 2,
"size": "15",
"wear": 59.94265362733487,
"worn": false
},
{
"flat": false,
"pos": 3,
"size": "15",
"wear": 38.2541625477213,
"worn": false
}
]
}
}
}
]
}
]
Tab Complete
Called “prompt” mode in gnmic.
6 - Ansible
Configure any RESTCONF client from ansible
Ansible can be used to get, set and call functions in RESTCONF. You can use the Ansible RESTCONF module or you can use the Ansible uri module to make RESTCONF calls as I’ve done here.
Setup
Downloading FreeCONF example source code
git clone https://github.com/freeconf/examples fc-examples
Setup ansible
cd ansible
python3 -m venv venv
source ./venv/bin/activate
python -m pip install -r requirements.txt
file: requirements.txt
ansible==7.0.0
ansible-core==2.15.8
Running
Inventory file
all:
hosts:
car:
http_port: 8090
ansible_host: localhost
Get Config
Command: ansible-playbook -i hosts.yml get-config.yml
File: get-config.yml
---
- name: get configure
hosts: car
connection: httpapi
gather_facts: False
tasks:
- name: get current speed
uri:
url: "http://{{ ansible_host }}:{{ http_port }}/restconf/data/car:?depth=1"
register: results
- debug:
var: results.json
Example Output:
****
TASK [get current speed] *******************************************************
ok: [car]
TASK [debug] *******************************************************************
ok: [car] => {
"results.json": {
"lastRotation": 13100,
"miles": 20900,
"running": false,
"speed": 300
}
}
PLAY RECAP *********************************************************************
car : ok=2 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
Set Config
Command: ansible-playbook -i hosts.yml set-config.yml
File: set-config.yml
---
- name: configure
hosts: car
connection: httpapi
gather_facts: False
tasks:
- name: change speed
uri:
url: "http://{{ ansible_host }}:{{ http_port }}/restconf/data/car:"
method: PATCH
body_format: json
body: |
{
"speed":10
}
Run RPCs
Here we run the replaceTires RPC
---
- name: rpc
hosts: car
connection: httpapi
gather_facts: False
tasks:
- name: replace tires
uri:
url: "http://{{ ansible_host }}:{{ http_port }}/restconf/data/car:replaceTires"
method: POST
7 - InfluxDB
Sending metrics to InfluxDB
Demonstrates:
- How to use export FreeCONF application metrics to InfluxDB w/o coupling your application code from InfluxDB
Details
The fcinflux.Service walks thru all the local management interfaces and auto-discovers all metrics (i.e. config false in YANG) and sends them to InfluxDB at a configurable frequency.
Inside the code
“Walking” the metrics.
FreeCONF API to recursively send metrics to InfluxDB.
package fcinflux
import (
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
"github.com/freeconf/yang/val"
)
// nodeWtr implements the FreeCONF node interface and acts like a "writer"
// when traversing a node tree with Selection.UpsertInto(<thisWriter>). There
// are other ways to accomplish walking a tree but this seemed easiest.
func nodeWtr(s sink, m Metric) node.Node {
fields := make(map[string]interface{})
return &nodeutil.Basic{
OnField: func(r node.FieldRequest, h *node.ValueHandle) error {
fields[r.Meta.Ident()] = h.Val.Value()
return nil
},
OnChild: func(r node.ChildRequest) (node.Node, error) {
return nodeWtr(s, m), nil
},
OnNext: func(r node.ListRequest) (node.Node, []val.Value, error) {
return nodeWtr(s, m), r.Key, nil
},
OnEndEdit: func(r node.NodeRequest) error {
if len(fields) > 0 {
m.Name = r.Selection.Path.String()
return s.send(r.Selection.Context, m, fields)
}
return nil
},
}
}
Running the example
Setting up InfluxDB
docker run -d --name=influxdb -p 8086:8086 influxdb:2.7
Configure and grab API token
Open a web browser to Influx DB admin portal at http://localhost:8086.
Created and admin account, organization and initial database.
Start Up Car Application
cd fcinflux/cmd
API_TOKEN=<api token from clipboard> go run .
tip: If you lost your API Token or ready to use a token for production, just log into Influx and generate a new one in menu.
Graph data
Log into InfluxDB interface and graph the data.
Conclusion
Comparison to gNMIc approach
This has advantage of working with all gNMIc compliant devices.
In general, FreeCONF is a library to build your own solutions, gNMIc is a utility to use in production as is. You can decide at any point which approach works for you without having to change your application code, just your deployment strategy.
Using this example code
If you wanted to use the approach here as is, you could import fcinflux into your application directly by calling go get github.com/freeconf/example but this example code may change without notice. It’s intention is to give you a starter project to customize as needed.
Expanding on this example code
Open a discussion about what you’d like to see or feel free to make an annoucement on what you built.
Ideal architecture:
8 - Prometheus
Sending metrics to Prometheus
You have a few options for integrating your metrics with Prometheus. You can use gNMIc or freeconf/example/fcprom module. This document explores the later.
Demonstrates:
- How to use export FreeCONF application metrics to Prometheus w/o coupling your application code to Prometheus
- How to use YANG extensions to improve to Prometheus results
Details
The fcprom.Bridge walks thru all the local management interfaces and auto-discovers all metrics (i.e. config false in YANG) and makes them available to Prometheus.
Because YANG doesn’t understand Prometheus’ metrics types like gauge or counter or know how you want to flatten metrics in YANG lists, the fcprom module uses YANG extensions to help you control the translation. YANG extensions are ignored by other systems.
Defining some extensions
module metrics-extension {
prefix "m";
namespace "freeconf.org";
description "Classify metrics into certain types so tools like Prometheus can handle them correctly";
revision 0000-00-00;
extension gauge {
description "a value that goes up and down. this is the default type if not specfied";
}
extension counter {
description "a value that only goes up and is always positive. Roll over is normal as well as reset on restart";
}
extension multivariate {
description "one or more fields (from same node) to use values as label in metric report";
}
}
Using our extensions
module car {
description "Car goes beep beep";
revision 2023-03-27;
namespace "freeconf.org";
prefix "car";
import metrics-extension {
prefix "metric";
}
leaf running {
type boolean;
config false;
}
leaf speed {
description "How fast the car goes";
type int32;
units milesPerSecond;
default 1000;
metric:counter;
}
leaf miles {
description "How many miles has car moved";
config false;
type decimal64 {
fraction-digits 2;
}
}
leaf lastRotation {
type int64;
config false;
}
list tire {
description "Rubber circular part that makes contact with road";
key "pos";
// used to help flatten metrics in lists
metric:multivariate;
leaf pos {
type int32;
}
leaf size {
type string;
default 15;
}
leaf worn {
config false;
type boolean;
}
leaf wear {
config false;
type decimal64 {
fraction-digits 2;
}
}
leaf flat {
config false;
type boolean;
}
action replace {
description "replace just this tire";
}
}
container engine {
anydata specs;
}
rpc reset {
description "Reset the odometer";
}
rpc rotateTires {
description "Rotate tires for optimal wear";
}
rpc replaceTires {
description "Replace all tires";
}
notification update {
description "Important state information about your car";
leaf event {
type enumeration {
enum carStarted {
value 1;
}
enum carStopped;
enum flatTire;
}
}
}
}
Running the example
Downloading FreeCONF example source code
git clone https://github.com/freeconf/examples fc-examples
Setup and Run Prometheues
1.) Download and install Prometheus
2.) Start Prometheus with the example configuration here.
cd fcprom
prometheus --config.file=prometheus.yml
file: prometheus.yml
global:
scrape_interval: 15s
evaluation_interval: 15s
rule_files:
# - "first.rules"
# - "second.rules"
scrape_configs:
- job_name: prometheus
static_configs:
- targets: ['localhost:8090']
Running Application
cd fcprom/cmd
go run .
Render Graph
Go to http://localhost:9090/ and enter car_tire_wear as expression
Using fcprom in your applications
package main
import (
"flag"
"log"
"strings"
"github.com/freeconf/examples/car"
"github.com/freeconf/examples/fcprom"
"github.com/freeconf/restconf"
"github.com/freeconf/restconf/device"
"github.com/freeconf/yang/source"
)
// Connect everything together into a server to start up
func main() {
flag.Parse()
// Your app here
app := car.New()
// where the yang files are stored
ypath := source.Path("../../yang:..")
// Device is just a container for browsers. Needs to know where YANG files are stored
d := device.New(ypath)
// Device can hold multiple modules, here we are only adding one
if err := d.Add("car", car.Manage(app)); err != nil {
panic(err)
}
// Prometheus will look at all local modules which will be car and fc-restconf
// unless configured to ignore the module
p := fcprom.NewBridge(d)
if err := d.Add("fc-prom", fcprom.Manage(p)); err != nil {
panic(err)
}
// Select wire-protocol RESTCONF to serve the device.
restconf.NewServer(d)
// apply start-up config normally stored in a config file on disk
config := `{
"fc-restconf":{
"web":{
"port":":8090"
}
},
"fc-prom" : {
"service" : {
"useLocalServer" : true
}
},
"car":{"speed":10}
}`
if err := d.ApplyStartupConfig(strings.NewReader(config)); err != nil {
panic(err)
}
if !*testMode {
// wait for ctrl-c
log.Printf("server started")
select {}
}
}
var testMode = flag.Bool("test", false, "do not run in background (i.e. driven by unit test)")
Conclusion
Comparison to gNMIc approach
This has advantage of working with all gNMIc compliant devices.
In general, FreeCONF is a library to build your own solutions, gNMIc is a utility to use in production as is. You can decide at any point which approach works for you without having to change your application code, just your deployment strategy.
Using this example code
If you wanted to use the approach here as is, you could import fcprom into your application directly by calling go get github.com/freeconf/example but this example code may change without notice. It’s intention is to give you a starter project to customize as needed.
Expanding on this example code
Open a discussion about what you’d like to see or feel free to make an annoucement on what you built.
Ideal architecture:
Tips for extending:
- Consider adding more YANG extensions like
metrics:ignoreto skip data ormetrics:labelto override the default label.
9 - Slack
Send notifications to Slack
Demonstrates
- How to send YANG notifications to Slack
Details
The fcslack.Service will selectively subscribe to a list of paths in the local application and send the event payload to a designated channel in Slack.
If any of the paths are invalid, you will get an error. This is good. Unlike scraping logs, if you don’t have your regex correct, you silently miss the event.
This example can be easily extended to send events to Kafka, AWS SQS, Pager Duty, etc.
Running the example
Downloading FreeCONF example source code
git clone https://github.com/freeconf/examples fc-examples
Setup Slack workspace, application and bot token
You will have to setup a Slack Application, register the application with your Slack workspace and generate a bot token.
Create an environment file with your settings
file : fcslack/cmd/env.sh
# Should start with xoxb- which means it is a "Bot token"
export SLACK_API_TOKEN=xoxb-5555555555555-5555555555555-5xyZ5xyZ5xyZ5xyZ5xyZ5xyZ
# Any channel you want, you can control each message if you want in your application
export SLACK_CHANNEL=testing-integration
Run car application with your settings
cd fcslack/cmd
source ./env.sh
go run .
You should see the {"event":"carStarted"} event right away in your slack channel, then in minute or so see the {"event":"flatTire"} and the {"event":"carStopped"} events.
Conclusion
FreeCONF is a library to build your own solutions. This code can work with any models so once you build it once you can use it everywhere. Also, you can have multiple systems subscribing to the same event streams so maybe you start with slack and then weave in PagerDuty integration once you’re confident this is a reliable event stream.
Using this example code
If you wanted to use the approach here as is, you could import fcslack into your application directly by calling go get github.com/freeconf/example but this example code may change without notice. It’s intention is to give you a starter project to customize as needed.
Ideas of Expanding
- Have a yang extension that will require fcslack to subscribe to ensure that event is always sent.
main.go
package main
import (
"flag"
"fmt"
"log"
"os"
"strings"
"github.com/freeconf/examples/car"
"github.com/freeconf/restconf"
"github.com/freeconf/restconf/device"
"github.com/freeconf/yang/source"
"github.com/freeconf/examples/fcslack"
)
func main() {
flag.Parse()
// Your app here
app := car.New()
// where the yang files are stored
ypath := source.Path("../../yang:../../car:..")
// Device is just a container for browsers. Needs to know where YANG files are stored
d := device.New(ypath)
// Device can hold multiple modules, here we are only adding one
if err := d.Add("car", car.Manage(app)); err != nil {
panic(err)
}
s := fcslack.NewService(d)
if err := d.Add("fc-slack", fcslack.Manage(s)); err != nil {
panic(err)
}
restconf.NewServer(d)
// apply start-up config normally stored in a config file on disk
config := fmt.Sprintf(`{
"fc-restconf":{
"debug": true,
"web":{
"port":":8090"
}
},
"fc-slack" : {
"client" : {
"apiToken": "%s"
},
"subscription": [
{
"channel" : "%s",
"module": "car",
"path": "update"
}
]
},
"car":{"speed":100}
}`, os.Getenv("SLACK_API_TOKEN"), os.Getenv("SLACK_CHANNEL"))
// bootstrap config for all local modules
if err := d.ApplyStartupConfig(strings.NewReader(config)); err != nil {
panic(err)
}
if !*testMode {
// wait for ctrl-c
log.Printf("server started")
select {}
}
}
var testMode = flag.Bool("test", false, "do not run in background (i.e. driven by unit test)")
10 - Node/Reflect
When your application objects and field names mostly align with your YANG model
Use cases:
- CRUD on Go structs
- CRUD on Go maps or slices
Special notes
- You don’t need perfect alignment with Go field names and YANG to use
Reflect. LetReflectdo the heavy lifting for you and capture small variations by combining with Extend. To that end, do not expect magical powers fromReflectto coerse your custom field types to YANG types. - Currently
Reflectdoesn’t attempt to use reflection to implementnotificationsoractions/rpcsbut again, you can combineReflectwithExtend. - Names in YANG can be
camelCase,kabob-caseorsnake_caseinterchangablely. Your Go public field are obviously inCamelCase.
Simple example
When you happen to have perfect alignment of field names to names in YANG.
If you have application code like this…
package demo
type Contacts struct {
Me *User
Users []*User
Size int
HarmlessValue string
}
type User struct {
FullName string
}
…and YANG like this…
module contacts {
namespace "freeconf.org";
prefix "a";
container me {
// full-name, full_name or fullName all work
leaf full-name {
type string;
}
}
list users {
key full-name;
// could be grouping/uses here, doesn't matter
leaf full-name {
type string;
}
}
leaf size {
type int32;
}
}
…then your node code can be this.
package demo
import (
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
)
func manage(app *Contacts) node.Node {
return &nodeutil.Node{Object: app}
}
…and you test like this.
package demo
import (
"testing"
"github.com/freeconf/yang/fc"
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
"github.com/freeconf/yang/parser"
"github.com/freeconf/yang/source"
)
func TestManageContact(t *testing.T) {
app := &Contacts{}
ypath := source.Dir(".")
m := parser.RequireModule(ypath, "contacts")
b := node.NewBrowser(m, manage(app))
actual, err := nodeutil.WriteJSON(b.Root())
fc.AssertEqual(t, nil, err)
fc.AssertEqual(t, `{"size":0}`, actual)
}
Map example
Reflect also supports Go’s map interface. While this Go code’s lack of data structures that might make this difficult to use in Go, if you don’t need to handle this data in Go, then this is completely acceptable. Remember, you can add constraints to yang to ensure the data is validated properly.
If you have application code like this…
package demo
type JunkDrawer struct {
Info map[string]interface{}
}
|| …and YANG like this…
module junk-drawer {
container info {
leaf name {
type string;
}
container more {
leaf size {
type decimal64;
}
}
list stuff {
key anything;
leaf anything {
type enumeration {
enum one;
enum two;
}
}
}
}
}
…then your node code can be this.
package demo
import (
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
)
func manageJunkDrawer(app *JunkDrawer) node.Node {
return &nodeutil.Node{Object: app}
}
…and you test like this.
package demo
import (
"testing"
"github.com/freeconf/yang/fc"
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
"github.com/freeconf/yang/parser"
"github.com/freeconf/yang/source"
)
func TestManageJunk(t *testing.T) {
app := &JunkDrawer{Info: make(map[string]interface{})}
ypath := source.Dir(".")
m := parser.RequireModule(ypath, "junk-drawer")
b := node.NewBrowser(m, manageJunkDrawer(app))
actual, err := nodeutil.WriteJSON(b.Root())
fc.AssertEqual(t, nil, err)
fc.AssertEqual(t, `{"info":{}}`, actual)
}
tips:
- useful for building validated RESTCONF APIs quickly to be filled in later with structs
- good for handling a bulk set of configs
Field coersion
Reflect can do a decent job converting Go primatives to and from YANG leaf types: strings to numbers, numbers to enums, enums to strings, number types to other number types, etc.. If a conversion represents a loss of data or a type can cannot be convert safely, then an error is returned. To handle the conversion of these values yourself, you can use Extend or Reflect.OnField . Reflect.OnField is more suited over Extend when have a lot of fields of the same type that you want to reuse in a lot of places and not one-offs.
If you have application code like this…
package demo
import "time"
type Timely struct {
LastModified time.Time
}
…and YANG like this…
module timely {
namespace "freeconf.org";
prefix "t";
leaf lastModified {
type int64; // unix seconds
}
}
…then your node code can be this.
package demo
import (
"reflect"
"time"
"github.com/freeconf/yang/meta"
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
)
func manageTimely(t *Timely) node.Node {
timeType := reflect.TypeOf(&time.Time{})
return &nodeutil.Node{
Object: t,
OnRead: func(ref *nodeutil.Node, m meta.Definition, t reflect.Type, v reflect.Value) (reflect.Value, error) {
switch t {
case timeType:
at := v.Interface().(*time.Time)
return reflect.ValueOf(at.Unix()), nil
}
return v, nil
},
OnWrite: func(ref *nodeutil.Node, m meta.Definition, t reflect.Type, v reflect.Value) (reflect.Value, error) {
switch t {
case timeType:
return reflect.ValueOf(time.Unix(v.Int(), 0)), nil
}
return v, nil
},
}
}
Adhoc structs
Create an anonymous struct or a just a map. Useful for handing RPC requests or responses. Here we use it to create whole app.
package demo
import (
"testing"
"github.com/freeconf/yang/fc"
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
"github.com/freeconf/yang/parser"
)
// Load a YANG module from a string!
var mstr = `
module mystery {
namespace "freeconf.org";
prefix "m";
leaf name {
type string;
}
}
`
func TestMystery(t *testing.T) {
m, err := parser.LoadModuleFromString(nil, mstr)
fc.AssertEqual(t, nil, err)
t.Run("struct", func(t *testing.T) {
// create a node from an adhoc struct
app := struct {
Name string
}{
"john",
}
manage := &nodeutil.Node{Object: &app}
// verify works
b := node.NewBrowser(m, manage)
actual, err := nodeutil.WriteJSON(b.Root())
fc.AssertEqual(t, nil, err)
fc.AssertEqual(t, `{"name":"john"}`, actual)
})
t.Run("map", func(t *testing.T) {
// create a node from map
app := map[string]interface{}{"name": "mary"}
manage := &nodeutil.Node{Object: app}
// verify works
b := node.NewBrowser(m, manage)
actual, err := nodeutil.WriteJSON(b.Root())
fc.AssertEqual(t, nil, err)
fc.AssertEqual(t, `{"name":"mary"}`, actual)
})
}
11 - Node/Extend
When you want to make adjustments to an existing node’s implementation
Use cases:
- Few isolated changes to an existing node
- Wrap a CRUD node but customize editing operations, actions or notifications
Special notes
- If [
Reflect]({ { < relref “reflect” > } } ) was a cake and [Basic]({ { < relref “basic” > } } ) was the plate under the cake, thenExtendwould be the frosting. Extendis exactly likeBasicbut let’s you delegate anything to another node. So most ofBasic'sdocumentation also applies here.
Reflect with one exception
package demo
import "fmt"
type Bird struct {
Name string
X int
Y int
}
func (b *Bird) GetCoordinates() string {
return fmt.Sprintf("%d,%d", b.X, b.Y)
}
class Bird():
def __init__(self, name, x, y):
self.name = name
self.x = x
self.y = y
def coordinates(self):
return f'{self.x},{self.y}'
module bird {
namespace "freeconf.org";
prefix "b";
leaf name {
type string;
}
leaf location {
config false;
type string {
pattern "[0-9.]+,[0-9.]+";
}
}
}
package demo
import (
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
"github.com/freeconf/yang/val"
)
func manage(b *Bird) node.Node {
return &nodeutil.Extend{
Base: nodeutil.ReflectChild(b), // handles Name
OnField: func(p node.Node, r node.FieldRequest, hnd *node.ValueHandle) error {
switch r.Meta.Ident() {
case "location":
hnd.Val = val.String(b.GetCoordinates())
default:
// delegates to ReflectChild for name
return p.Field(r, hnd)
}
return nil
},
}
}
from freeconf import nodeutil, val
def manage_bird(bird):
base = nodeutil.Node(bird)
def on_field(parent, req, write_val):
if req.meta.ident == "location":
return val.Val(bird.coordinates())
return parent.field(req, write_val)
return nodeutil.Extend(base, on_field=on_field)
Addition Files
file: manage_test.go
package demo
import (
"testing"
"github.com/freeconf/yang/fc"
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
"github.com/freeconf/yang/parser"
"github.com/freeconf/yang/source"
)
func TestManage(t *testing.T) {
b := &Bird{Name: "sparrow", X: 99, Y: 1000}
ypath := source.Dir(".")
m := parser.RequireModule(ypath, "bird")
bwsr := node.NewBrowser(m, manage(b))
root := bwsr.Root()
defer root.Release()
actual, err := nodeutil.WriteJSON(root)
fc.AssertEqual(t, nil, err)
fc.AssertEqual(t, `{"name":"sparrow","location":"99,1000"}`, actual)
}
file: test_manage.py
#!/usr/bin/env python3
import unittest
from freeconf import parser, nodeutil, source, node
from manage import manage_bird
from bird import Bird
class TestManage(unittest.TestCase):
def test_manage(self):
app = Bird("sparrow", 99, 1000)
ypath = source.path('..')
m = parser.load_module_file(ypath, 'bird')
mgmt = manage_bird(app)
bwsr = node.Browser(m, mgmt)
root = bwsr.root()
try:
actual = nodeutil.json_write_str(root)
self.assertEqual('{"name":"sparrow","location":"99,1000"}', actual)
finally:
root.release()
if __name__ == '__main__':
unittest.main()
12 - Node/anydata
YANG
anydata can be used send a variety of valuesYANG has a type called anydata which can be anything. Reflect’s default behavior is to keep this as whatever the source node sends. For RESTCONF web requests this:
map[string]interface{}- When given loose datadecimal64- when a numberstring- when given a stringio.Reader- when given a file uploaded thruformmime type. See Forms
13 - Code generation
Writing code that writes code
While code generation is available now, this is a placeholder for the documentation that is coming soon.
In the meantime you can browse some of the early code here.
14 - Debugging
Techniques for debugging
Debug Logging
import (
"github.com/freeconf/yang/fc"
)
...
// turn on debug logging
fc.DebugLog(true)
Logging node activity
Use cases:
- See all operations performed on a node when you’re not sure you are getting the right data in or out.
Usage
// wrap all node activity to the node app recursively
n := nodeutil.Dump(manageApp(app), os.Stdout)
Example Edit
BeginEdit, new=false, src=car
->speed=int32(10)
EndEdit, new=false, src=car
Example Read
<-speed=int32(10)
<-miles=decimal64(310.000000)
tips:
- Take a look at the source and create your own dumper that is maybe better.
- You can do this at root node or any part of the tree you want to inspect
15 - Unit Testing
Strategies unit testing management
nodesYou don’t need any special utilities to unit test your management, just a few useful techniques. If you look thru a lot of the node implementations you’ll see a lot of unit tests exibiting how to test nodes.
Testing without full application
As your application, management and YANG file grow, loading the full application each time just to test a piece might become cumbersome. We can use a feature of YANG to import from another YANG file and FreeCONF’s ability to make this easy to test just our unit.
package demo
import (
"testing"
"github.com/freeconf/yang/fc"
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
"github.com/freeconf/yang/parser"
"github.com/freeconf/yang/source"
)
type Here struct {
Penny int
}
func TestUnitTestingPartialYang(t *testing.T) {
// where car.yang is stored
ypath := source.Dir(".")
// Define new YANG module on the fly that references the application
// YANG file but we pull in just what we want
m, err := parser.LoadModuleFromString(ypath, `
module x {
import deep {
prefix "d";
}
// pull in just the piece we are interested in. Here it is
// just a single penny
uses d:here;
}
`)
if err != nil {
t.Fatal(err)
}
// We create a "browser" to just a unit of our application
h := &Here{Penny: 1}
manage := node.NewBrowser(m, nodeutil.ReflectChild(h))
// Example : test getting config
actual, err := nodeutil.WriteJSON(manage.Root())
fc.AssertEqual(t, nil, err)
fc.AssertEqual(t, `{"penny":1}`, actual)
}
16 - Web UIs
Building a Web UI interface to a RESTCONF implementation is pure joy. This is a few tips that will serve you well.
While automation is primarily about APIs, building administration portals enable non-developers to interact with your application and another point of access for developers.
Car Admin Portal
Notes:
- This example web code is very basic, but you can use advanced web frameworks
- Read how to use RESTCONF API to understand how to navigate data.
- Serving your web interface with your application ensures they are always deployed together and therefore compatible.
// Basic calls to interact with the "car" RESTCONF API
let car = null;
// Reading config AND metrics together for the whole car
function update() {
fetch("/restconf/data/car:").then((resp) => {
return resp.json();
}).then((data) => {
car = data;
let e = document.getElementById("car");
e.innerHTML = `
<div>Miles: ${data.miles} miles</div>
<div>Running: ${data.running}</div>
<div>Speed: ${data.speed} mps</div>
${data.tire.map((tire, index) =>
`<div>Tire ${index}: Flat: ${tire.flat}, Wear: ${tire.wear}</div>`
).join('')}
`;
});
};
// Call one of the RPCs. ours are simple, no input or expected output but
// you could easily add them here
function run(action) {
fetch(`/restconf/data/car:${action}`, {
method: 'POST',
}).then(() => {
update();
});
}
// Update config edit.
// PATCH is like an upsert.
// PUT is like a delete and replace with this.
// POST is create new
function speed(s) {
fetch('/restconf/data/car:', {
method: 'PATCH',
body : JSON.stringify({
speed: car.speed + s
})
}).then(() => {
update();
});
}
// Subscribe to notifications. EventSource is part of every browser and works
// on HTTP (i.e. http) but more efficient on HTTP/2 (i.e. https)
function subscribeToUpdateEventStream() {
// this will appear as a pending request.
const events = new EventSource("/restconf/data/car:update?simplified");
events.addEventListener("message", e => {
// this decodes SSE events for you to give you just the messages
const log = document.getElementById("log");
log.appendChild(document.createTextNode(e.data + '\n'));
});
// to unsubscribe just close the stream.
// events.close();
}
// Metrics change too often for subscription so just get the latest
// every 2s.
const pollInterval = 2000;
function pollforUpdates() {
update();
setTimeout(pollforUpdates, pollInterval);
}
// initial read of data
pollforUpdates();
// watch for update events as they happen, no polling here
subscribeToUpdateEventStream();
<html><!DOCTYPE html>
<html lang="en">
<head>
<title>Car Demo</title>
<meta name="viewport" content="width=device-width, minimum-scale=1, initial-scale=1, user-scalable=yes">
</head>
<body>
<div>
<button onclick='speed(10)'>Faster</button>
<button onclick='speed(-10)'>Slower</button>
<button onclick='run("replaceTires")'>Replace Tires</button>
<button onclick='run("rotateTires")'>Rotate Tires</button>
<button onclick='run("reset")'>Reset Odometer</button>
</div>
<div id="car"></div>
<p><b>tip:</b> keep rotating the tires to see how long how many miles you can get without getting a flat.</p>
<script src="/ui/app.js"></script>
<h3>Log</h3>
<pre id="log"></pre>
</body>
</html>
Additional Notes:
- Register custom request endpoints - Not everything has to be handled thru RESTCONF. Use standard web request handlers for any custom endpoints.
- Use
notificationfor interactive UIs - Notifications aren’t just for alerts. One of the more useful notifications is for data has changed in back-end from possibly another user edit and front-end should reload data - Consider a web-only module - You can serve any number of modules in an application should you need to isolate your web-only functions. For example
carmodule andcar-webmodule both from the same server. - Generate REST API Docs - Generate REST API docs to know what is available from REST.
#!/usr/bin/env sh
set -eu
FC_YANG="go run github.com/freeconf/yang/cmd/fc-yang"
YPATH=../car
MODULE=car
# HTML API Docs
${FC_YANG} doc -f dot -module ${MODULE} -ypath ${YPATH} > ${MODULE}.dot
dot -Tsvg ${MODULE}.dot -o ${MODULE}.svg
${FC_YANG} doc -f html -module ${MODULE} -title "Car REST API" -ypath ${YPATH} > ${MODULE}-api.html
# Markdown API Docs
${FC_YANG} doc -f md -module ${MODULE} -ypath ${YPATH} > ${MODULE}-api.md
File Uploading
module file-upload {
namespace "freeconf.org";
prefix "f";
rpc bookReport {
input {
leaf bookName {
description "Name of the book";
type string;
}
anydata pdf {
description "PDF of file upload";
}
}
output {
leaf fileSize {
description "";
type uint64;
}
}
}
}
Frontend
<html><!DOCTYPE html>
<html lang="en">
<head>
<title>Book Report Upload</title>
<meta name="viewport" content="width=device-width, minimum-scale=1, initial-scale=1, user-scalable=yes">
</head>
<body>
<table>
<tr>
<td><label for="bookName">Book Name:</label></td>
<td><input id="bookName"></td>
</tr>
<tr>
<td><label for="pdf">Upload:</label></td>
<td><input type="file" id="pdf" onchange='upload()'></td>
</tr>
</table>
<script src="/ui/file-upload.js"></script>
</body>
</html>
// grab the form data and upload the file to RESTCONF rpc
function upload() {
const bookName = document.getElementById("bookName").value;
const pdf = document.getElementById("pdf").files[0];
const form = new FormData();
form.append("bookName", bookName); // YANG: leaf bookName { type string; }
form.append("pdf", pdf); // YANG: anydata pdf;
fetch("/restconf/data/file-upload:bookReport", {
method: "POST",
body: form
})
.then(resp => resp.json())
.then((data) => {
window.alert(`success. file size is ${data.fileSize}`);
});
}
Backend
Go will see the anydata as an io.Reader.
package main
import (
"io"
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
)
// matches rpc input of uploading a book report
type upload struct {
BookName string
Pdf io.Reader
}
// handle just one thing, file upload of book report
func manageUploader() node.Node {
return &nodeutil.Basic{
OnAction: func(r node.ActionRequest) (node.Node, error) {
switch r.Meta.Ident() {
case "bookReport":
// you should do more validation of input data
var req upload
r.Input.UpdateInto(nodeutil.ReflectChild(&req))
data, err := io.ReadAll(req.Pdf)
if err != nil {
return nil, err
}
// we throw away the report, but you'd probably want to keep it
// tell user the file size as proof we got it
resp := struct {
FileSize uint64
}{
FileSize: uint64(len(data)),
}
return nodeutil.ReflectChild(&resp), nil
}
return nil, nil
},
}
}
Model Driven UI
Being able to read the information in the YANG file from your web application is nothing short of game changing. Here are just a few of the possibilities:
- client-side form validation
- build forms dynamically including simple things like select list options from leaf enumerations
- form labels from leaf names and tooltips from descriptions
- list of available columns in a table
- reports
Combine this with ability to extend the YANG with your own meta data the possibilites are endless. For example:
- mark
leafsas password fields - marking fields that require web custom handlers
- fields that should be shown to advanced users
- fields that should only show if feature flag is on
The path to the meta definitions is just /restconf/schema/{module}/ and requires header Accept: application/json to return the YANG file in JSON form. You can use all normal RESTCONF navigation features to drill in to the just the part of the YANG file you are interested in.
<html><!DOCTYPE html>
<html lang="en">
<head>
<title>Car Model</title>
<meta name="viewport" content="width=device-width, minimum-scale=1, initial-scale=1, user-scalable=yes">
</head>
<body>
<h2>Edit Car</h2>
<div id="form"></div>
<script src="/ui/model-driven.js"></script>
</body>
</html>
// load the schema (YANG) and the object for editing
function load(module) {
const parent = document.getElementById("form");
Promise.allSettled([
fetch(`/restconf/data/${module}:`)
.then(resp => resp.json()),
fetch(`/restconf/schema/${module}/`, {
method: 'GET',
headers: {'Accept': 'application/json'}
}).then(resp => resp.json()),
]).then((responses) => {
render(parent, responses[0].value, responses[1].value.module);
});
}
// navigate thru the meta along with the object to build forms but
// as a pattern for all model driven UI
function render(parent, obj, meta) {
const editableFields = meta.dataDef.filter(def => {
// ignore mertics
if (def.leaf?.config == false) {
return false;
}
// would normally recurse here
if ('list' in def || 'container' in def) {
return false;
}
return true;
})
// here you would normally adjust the input type based on details
// in the 'def' object like number v.s. string, etc.
parent.innerHTML = `
<table>
${editableFields.map(def => `
<tr>
<td><label>${def.ident}</label></td>
<td><input value="${obj[def.ident] || ''}"></td>
</tr>
</table>`).join('')}
`;
// recurse into lists and containers here
}
// car is not a very exciting object to edit but still demonstrates feature
load('car');
17 - Request filters
How to register web request filters to read custom request headers and how to relay that information to your nodes.
RESTCONF HTTP request access
This is useful for custom authorization or general need to access the HTTP request information
Steps:
1.) Register a request restconf.RequestFilter with RESTCONF restconf.Server instance
2.) Filter returns a context.Context that contains any custom data that you might extract from the HTTP request like HTTP header information, URL parameters or certificate information.
3.) Values from that context.Context will be made available to all your node.Node implementations
Example Code:
package demo
import (
"context"
"io"
"net/http"
"strings"
"testing"
"github.com/freeconf/restconf"
"github.com/freeconf/restconf/device"
"github.com/freeconf/yang/fc"
"github.com/freeconf/yang/node"
"github.com/freeconf/yang/nodeutil"
"github.com/freeconf/yang/parser"
"github.com/freeconf/yang/source"
"github.com/freeconf/yang/val"
)
type App struct {
}
var module = `module x {
namespace "freeconf.org";
prefix "x";
leaf messageFromRequest {
config false;
type string;
}
}
`
type testContextKey string
const contextKey = testContextKey("requestKey")
func manageApp(a *App) node.Node {
return &nodeutil.Basic{
OnField: func(r node.FieldRequest, hnd *node.ValueHandle) error {
switch r.Meta.Ident() {
case "messageFromRequest":
msg := r.Selection.Context.Value(contextKey).(string)
hnd.Val = val.String(msg)
}
return nil
},
}
}
func startServer() *restconf.Server {
ypath := source.Any(restconf.InternalYPath, source.Dir("../yang"))
m, err := parser.LoadModuleFromString(ypath, module)
if err != nil {
panic(err)
}
app := &App{}
b := node.NewBrowser(m, manageApp(app))
d := device.New(ypath)
d.AddBrowser(b)
s := restconf.NewServer(d)
// Register a filter to peek at request. From here you can look at:
// * headers
// * certs
// * url params
grabHeader := func(ctx context.Context, w http.ResponseWriter, r *http.Request) (context.Context, error) {
msg := r.Header.Get("X-MESSAGE")
// stuff your data into the context and it will be available to all node
// navigation associated with this request.
ctx = context.WithValue(ctx, contextKey, msg)
return ctx, nil
}
s.Filters = append(s.Filters, grabHeader)
d.ApplyStartupConfig(strings.NewReader(`{
"fc-restconf" : {
"web" : {
"port" : ":9999"
}
}
}
`))
return s
}
func TestRequestAccess(t *testing.T) {
startServer()
t.Run("request", func(t *testing.T) {
req, err := http.NewRequest("GET", "http://localhost:9999/restconf/data/x:", nil)
fc.AssertEqual(t, nil, err)
req.Header.Add("X-MESSAGE", "hi")
c := &http.Client{}
resp, err := c.Do(req)
fc.AssertEqual(t, nil, err)
fc.AssertEqual(t, 200, resp.StatusCode)
actual, err := io.ReadAll(resp.Body)
fc.AssertEqual(t, nil, err)
fc.AssertEqual(t, `{"messageFromRequest":"hi"}`, string(actual))
})
}