Convert to async MIB instrumentation API (#209)

MIB instrumentation API changed to allow for asynchronous
managed objects access. Although built-in SNMPv2-SMI objects
are still synchronous, the MIB instrumentation API is async
what allows users to replace default MIB instrumentation
with their own, potentially asynchronous.

CommandResponder refactored to facilitate asynchronous
MIB instrumentation routines. The `readVars`, `readNextVars`
and `writeVars` MIB controller methods return immediately and
deliver their results via a call back.

SMI/MIB managed objects API overhauled for simplicity and
flexibility breaking backward compatibility.

CHANGES.txt

@@ -5,9 +5,11 @@ Revision 5.0.0, released 2018-10-??
 - SNMPv3 crypto operations that require external dependencies
   made dependent on the optional external
   package -- pysnmpcrypto.
+
 - By switching to pysnmpcrypto, pysnmp effectively migrates from
   PyCryptodomex to pyca/cryptography whenever available on the
   platform.
+
 - Many really old backward-compatibility code snippets removed.
   Most importantly:
   
@@ -19,13 +21,20 @@ Revision 5.0.0, released 2018-10-??
 - The MIB instrumentation API overhauled in backward incompatible
   way:
 
-      - MIB instrumentation methods signatures simplified to accept
+  * MIB instrumentation methods signatures simplified to accept
-        just var-binds (as var-arg), the rest of the parameters packed
+    just var-binds (as var-arg), the rest of the parameters packed
-        into opaque kwargs
+    into opaque kwargs
-      - CommandResponder application passes `snmpEngine` and optionally
+
-        user-supplied `cbCtx` object throughout the MIB instrumentation
+  * CommandResponder application passes `snmpEngine` and optionally
-        methods. The goal is to let MIB objects access/modify whatever
+    user-supplied `cbCtx` object throughout the MIB instrumentation
-        custom Python objects they need while being called back.
+    methods. The goal is to let MIB objects access/modify whatever
+    custom Python objects they need while being called back.
+
+  * CommandResponder refactored to facilitate asynchronous
+    MIB instrumentation routines. The `readVars`, `readNextVars` and
+    `writeVars` MIB controller methods return immediately and
+    deliver their results via a call back.
+
 - The high-level API (`hlapi`) extended to cover lightweight SNMP v1arch
   in hope to ease the use of packet-level SNMP API.
 
@@ -41,6 +50,7 @@ Revision 5.0.0, released 2018-10-??
   automation around building well-formed SNMP messages is and mediating
   differences between SNMP versions is not present in this new `v1arch`
   layer.
+
 - The signature of the hlapi `.sendNotification()` call has changed
   to accept `*varBinds` instead of a sequence of `varBinds`. The rationale
   is to unify this method call with similar methods of CommandGenerator.

examples/smi/agent/custom-managed-object.py

@@ -51,13 +51,23 @@ if __name__ == '__main__':
 
     mibInstrum = instrum.MibInstrumController(mibBuilder)
 
+    def cbFun(varBinds, **context):
+        for oid, val in varBinds:
+            if exval.endOfMib.isSameTypeWith(val):
+                context['state']['stop'] = True
+            print('%s = %s' % ('.'.join([str(x) for x in oid]), not val.isValue and 'N/A' or val.prettyPrint()))
+
+        context['state']['varBinds'] = varBinds
+
+    context = {
+        'cbFun': cbFun,
+        'state': {
+            'varBinds': [((1, 3, 6), None)],
+            'stop': False
+        }
+    }
+
     print('Remote manager read access to MIB instrumentation (table walk)')
-
+    while not context['state']['stop']:
-    varBinds = [((), None)]
+        mibInstrum.readNextVars(*context['state']['varBinds'], **context)
-
+    print('done')
-    while True:
-        varBinds = mibInstrum.readNextVars(*varBinds)
-        oid, val = varBinds[0]
-        if exval.endOfMib.isSameTypeWith(val):
-            break
-        print(oid, val.prettyPrint())

examples/smi/agent/operations-on-managed-objects.py

@@ -26,24 +26,40 @@ snmpCommunityEntry, = mibBuilder.importSymbols(
 instanceId = snmpCommunityEntry.getInstIdFromIndices('my-router')
 print('done')
 
+
+def cbFun(varBinds, **context):
+    for oid, val in varBinds:
+        print('%s = %s' % ('.'.join([str(x) for x in oid]), not val.isValue and 'N/A' or val.prettyPrint()))
+
 print('Create/update SNMP-COMMUNITY-MIB::snmpCommunityEntry table row: ')
-varBinds = mibInstrum.writeVars(
+mibInstrum.writeVars(
     (snmpCommunityEntry.name + (2,) + instanceId, 'mycomm'),
     (snmpCommunityEntry.name + (3,) + instanceId, 'mynmsname'),
-    (snmpCommunityEntry.name + (7,) + instanceId, 'volatile')
+    (snmpCommunityEntry.name + (7,) + instanceId, 'volatile'),
+    cbFun=cbFun
 )
-for oid, val in varBinds:
-    print('%s = %s' % ('.'.join([str(x) for x in oid]), not val.isValue and 'N/A' or val.prettyPrint()))
 print('done')
 
+
+def cbFun(varBinds, **context):
+    for oid, val in varBinds:
+        if exval.endOfMib.isSameTypeWith(val):
+            context['state']['stop'] = True
+        print('%s = %s' % ('.'.join([str(x) for x in oid]), not val.isValue and 'N/A' or val.prettyPrint()))
+
+    context['state']['varBinds'] = varBinds
+
+context = {
+    'cbFun': cbFun,
+    'state': {
+        'varBinds': [((1, 3, 6), None)],
+        'stop': False
+    }
+}
+
 print('Read whole MIB (table walk)')
-varBinds = [((), None)]
+while not context['state']['stop']:
-while True:
+    mibInstrum.readNextVars(*context['state']['varBinds'], **context)
-    varBinds = mibInstrum.readNextVars(*varBinds)
-    oid, val = varBinds[0]
-    if exval.endOfMib.isSameTypeWith(val):
-        break
-    print('%s = %s' % ('.'.join([str(x) for x in oid]), not val.isValue and 'N/A' or val.prettyPrint()))
 print('done')
 
 print('Unloading MIB modules...'),

examples/v3arch/asyncore/agent/cmdrsp/custom-mib-controller.py

@@ -54,7 +54,9 @@ snmpContext = context.SnmpContext(snmpEngine)
 # always echos request var-binds in response.
 class EchoMibInstrumController(instrum.AbstractMibInstrumController):
     def readVars(self, *varBinds, **context):
-        return [(ov[0], v2c.OctetString('You queried OID %s' % ov[0])) for ov in varBinds]
+        cbFun = context.get('cbFun')
+        if cbFun:
+            cbFun([(ov[0], v2c.OctetString('You queried OID %s' % ov[0])) for ov in varBinds], **context)
 
 
 # Create a custom Management Instrumentation Controller and register at

pysnmp/entity/config.py

@@ -422,7 +422,8 @@ def delContext(snmpEngine, contextName):
     vacmContextEntry, tblIdx = __cookVacmContextInfo(snmpEngine, contextName)
 
     snmpEngine.msgAndPduDsp.mibInstrumController.writeVars(
-        (vacmContextEntry.name + (2,) + tblIdx, 'destroy')
+        (vacmContextEntry.name + (2,) + tblIdx, 'destroy'),
+        ** dict(snmpEngine=snmpEngine)
     )
 
 

pysnmp/entity/rfc3413/cmdrsp.py

@@ -25,8 +25,7 @@ class CommandResponderBase(object):
         self.cbCtx = cbCtx
         self.__pendingReqs = {}
 
-    def handleMgmtOperation(self, snmpEngine, stateReference,
+    def initiateMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
-                            contextName, PDU, acCtx):
         pass
 
     def close(self, snmpEngine):
@@ -147,7 +146,7 @@ class CommandResponderBase(object):
             'processPdu: stateReference %s, varBinds %s' % (stateReference, varBinds))
 
         try:
-            self.handleMgmtOperation(snmpEngine, stateReference, contextName, PDU)
+            self.initiateMgmtOperation(snmpEngine, stateReference, contextName, PDU)
 
         # SNMPv2 SMI exceptions
         except pysnmp.smi.error.GenError:
@@ -263,53 +262,79 @@ class CommandResponderBase(object):
 class GetCommandResponder(CommandResponderBase):
     pduTypes = (rfc1905.GetRequestPDU.tagSet,)
 
+    def completeMgmtOperation(self, varBinds, **context):
+        self.sendVarBinds(context['snmpEngine'], context['stateReference'],
+                          0, 0, varBinds)
+        self.releaseStateInformation(context['stateReference'])
+
     # rfc1905: 4.2.1
-    def handleMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
+    def initiateMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
         # rfc1905: 4.2.1.1
         mgmtFun = self.snmpContext.getMibInstrum(contextName).readVars
         varBinds = v2c.apiPDU.getVarBinds(PDU)
 
-        context = dict(snmpEngine=snmpEngine, acFun=self.verifyAccess, cbCtx=self.cbCtx)
+        context = dict(snmpEngine=snmpEngine,
+                       stateReference=stateReference,
+                       acFun=self.verifyAccess,
+                       cbFun=self.completeMgmtOperation,
+                       cbCtx=self.cbCtx)
 
-        rspVarBinds = mgmtFun(*varBinds, **context)
+        mgmtFun(*varBinds, **context)
-
-        self.sendVarBinds(snmpEngine, stateReference, 0, 0, rspVarBinds)
-        self.releaseStateInformation(stateReference)
 
 
 class NextCommandResponder(CommandResponderBase):
     pduTypes = (rfc1905.GetNextRequestPDU.tagSet,)
 
+    def completeMgmtOperation(self, varBinds, **context):
+        self.sendVarBinds(context['snmpEngine'], context['stateReference'],
+                          0, 0, varBinds)
+        self.releaseStateInformation(context['stateReference'])
+
     # rfc1905: 4.2.2
-    def handleMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
+    def initiateMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
         # rfc1905: 4.2.2.1
         mgmtFun = self.snmpContext.getMibInstrum(contextName).readNextVars
 
         varBinds = v2c.apiPDU.getVarBinds(PDU)
 
-        context = dict(snmpEngine=snmpEngine, acFun=self.verifyAccess, cbCtx=self.cbCtx)
+        context = dict(snmpEngine=snmpEngine,
+                       stateReference=stateReference,
+                       acFun=self.verifyAccess,
+                       cbFun=self.completeMgmtOperation,
+                       cbCtx=self.cbCtx)
 
-        while True:
+        mgmtFun(*varBinds, **context)
-            rspVarBinds = mgmtFun(*varBinds, **context)
-
-            try:
-                self.sendVarBinds(snmpEngine, stateReference, 0, 0, rspVarBinds)
-
-            except error.StatusInformation:
-                idx = sys.exc_info()[1]['idx']
-                varBinds[idx] = (rspVarBinds[idx][0], varBinds[idx][1])
-            else:
-                break
-
-        self.releaseStateInformation(stateReference)
 
 
 class BulkCommandResponder(CommandResponderBase):
     pduTypes = (rfc1905.GetBulkRequestPDU.tagSet,)
     maxVarBinds = 64
 
+    def _completeNonRepeaters(self, varBinds, **context):
+        context['rspVarBinds'][:] = varBinds
+        context['cbFun'] = self.completeMgmtOperation
+
+        mgmtFun = self.snmpContext.getMibInstrum(context['contextName']).readNextVars
+
+        mgmtFun(*context['varBinds'], **context)
+
+    def completeMgmtOperation(self, varBinds, **context):
+        context['rspVarBinds'].extend(varBinds)
+        context['counters']['M'] -= 1
+
+        if context['counters']['M'] and context['counters']['R']:
+            mgmtFun = self.snmpContext.getMibInstrum(context['contextName']).readNextVars
+
+            context['cbFun'] = self.completeMgmtOperation
+            mgmtFun(*varBinds[-context['counters']['R']:], **context)
+
+        else:
+            self.sendVarBinds(context['snmpEngine'], context['stateReference'],
+                              0, 0, varBinds)
+            self.releaseStateInformation(context['stateReference'])
+
     # rfc1905: 4.2.3
-    def handleMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
+    def initiateMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
         nonRepeaters = v2c.apiBulkPDU.getNonRepeaters(PDU)
         if nonRepeaters < 0:
             nonRepeaters = 0
@@ -318,68 +343,60 @@ class BulkCommandResponder(CommandResponderBase):
         if maxRepetitions < 0:
             maxRepetitions = 0
 
-        reqVarBinds = v2c.apiPDU.getVarBinds(PDU)
+        varBinds = v2c.apiPDU.getVarBinds(PDU)
 
-        N = min(int(nonRepeaters), len(reqVarBinds))
+        N = min(int(nonRepeaters), len(varBinds))
         M = int(maxRepetitions)
-        R = max(len(reqVarBinds) - N, 0)
+        R = max(len(varBinds) - N, 0)
 
         if R:
             M = min(M, self.maxVarBinds // R)
 
-        debug.logger & debug.flagApp and debug.logger('handleMgmtOperation: N %d, M %d, R %d' % (N, M, R))
+        debug.logger & debug.flagApp and debug.logger(
+            'initiateMgmtOperation: N %d, M %d, R %d' % (N, M, R))
 
         mgmtFun = self.snmpContext.getMibInstrum(contextName).readNextVars
 
-        context = dict(snmpEngine=snmpEngine, acFun=self.verifyAccess, cbCtx=self.cbCtx)
+        context = dict(snmpEngine=snmpEngine,
+                       stateReference=stateReference,
+                       contextName=contextName,
+                       acFun=self.verifyAccess,
+                       cbFun=self._completeNonRepeaters,
+                       cbCtx=self.cbCtx,
+                       varBinds=varBinds[-R:],
+                       counters={'M': M, 'R': R},
+                       rspVarBinds=[])
 
-        if N:
+        mgmtFun(*varBinds[:N], **context)
-            # TODO(etingof): manage all PDU var-binds in a single call
-            rspVarBinds = mgmtFun(*reqVarBinds[:N], **context)
-
-        else:
-            rspVarBinds = []
-
-        varBinds = reqVarBinds[-R:]
-
-        while M and R:
-            rspVarBinds.extend(mgmtFun(*varBinds, **context))
-            varBinds = rspVarBinds[-R:]
-            M -= 1
-
-        if len(rspVarBinds):
-            self.sendVarBinds(snmpEngine, stateReference, 0, 0, rspVarBinds)
-            self.releaseStateInformation(stateReference)
-        else:
-            raise pysnmp.smi.error.SmiError()
 
 
 class SetCommandResponder(CommandResponderBase):
     pduTypes = (rfc1905.SetRequestPDU.tagSet,)
 
+    def completeMgmtOperation(self, varBinds, **context):
+        self.sendVarBinds(context['snmpEngine'], context['stateReference'],
+                          0, 0, varBinds)
+        self.releaseStateInformation(context['stateReference'])
+
     # rfc1905: 4.2.5
-    def handleMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
+    def initiateMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
         mgmtFun = self.snmpContext.getMibInstrum(contextName).writeVars
 
         varBinds = v2c.apiPDU.getVarBinds(PDU)
 
-        instrumError = None
+        context = dict(snmpEngine=snmpEngine,
-
+                       stateReference=stateReference,
-        context = dict(snmpEngine=snmpEngine, acFun=self.verifyAccess, cbCtx=self.cbCtx)
+                       acFun=self.verifyAccess,
+                       cbFun=self.completeMgmtOperation,
+                       cbCtx=self.cbCtx)
 
         # rfc1905: 4.2.5.1-13
         try:
-            rspVarBinds = mgmtFun(*varBinds, **context)
+            mgmtFun(*varBinds, **context)
 
         except (pysnmp.smi.error.NoSuchObjectError,
                 pysnmp.smi.error.NoSuchInstanceError):
             instrumError = pysnmp.smi.error.NotWritableError()
             instrumError.update(sys.exc_info()[1])
-
+            self.releaseStateInformation(stateReference)
-        else:
-            self.sendVarBinds(snmpEngine, stateReference, 0, 0, rspVarBinds)
-
-        self.releaseStateInformation(stateReference)
-
-        if instrumError:
             raise instrumError

pysnmp/smi/instrum.py

@@ -184,15 +184,25 @@ class MibInstrumController(AbstractMibInstrumController):
     # MIB instrumentation
 
     def flipFlopFsm(self, fsmTable, *varBinds, **context):
-        self.__indexMib()
 
-        debug.logger & debug.flagIns and debug.logger('flipFlopFsm: input var-binds %r' % (varBinds,))
+        try:
+            fsmContext = context['fsmState']
+
+        except KeyError:
+            self.__indexMib()
+
+            fsmContext = context['fsmState'] = dict(varBinds=[], state='start', status='ok')
+
+            debug.logger & debug.flagIns and debug.logger('flipFlopFsm: input var-binds %r' % (varBinds,))
 
         mibTree, = self.mibBuilder.importSymbols('SNMPv2-SMI', 'iso')
 
-        outputVarBinds = []
+        outputVarBinds = fsmContext['varBinds']
-        state, status = 'start', 'ok'
+        state = fsmContext['state']
+        status = fsmContext['status']
+
         origExc = origTraceback = None
+
         while True:
             k = state, status
             if k in fsmTable:
@@ -237,7 +247,7 @@ class MibInstrumController(AbstractMibInstrumController):
                     break
                 else:
                     debug.logger & debug.flagIns and debug.logger(
-                        'flipFlopFsm: fun %s suceeded for %s=%r' % (mgmtFun, name, val))
+                        'flipFlopFsm: fun %s succeeded for %s=%r' % (mgmtFun, name, val))
                     if rval is not None:
                         outputVarBinds.append((rval[0], rval[1]))
 
@@ -252,13 +262,15 @@ class MibInstrumController(AbstractMibInstrumController):
                     # (seems to be irrelevant on Py3 but just in case)
                     del origTraceback
 
-        return outputVarBinds
+        cbFun = context.get('cbFun')
+        if cbFun:
+            cbFun(outputVarBinds, **context)
 
     def readVars(self, *varBinds, **context):
-        return self.flipFlopFsm(self.fsmReadVar, *varBinds, **context)
+        self.flipFlopFsm(self.fsmReadVar, *varBinds, **context)
 
     def readNextVars(self, *varBinds, **context):
-        return self.flipFlopFsm(self.fsmReadNextVar, *varBinds, **context)
+        self.flipFlopFsm(self.fsmReadNextVar, *varBinds, **context)
 
     def writeVars(self, *varBinds, **context):
-        return self.flipFlopFsm(self.fsmWriteVar, *varBinds, **context)
+        self.flipFlopFsm(self.fsmWriteVar, *varBinds, **context)