Merge branch 'async-mib-instrumentation' of github.com:etingof/pysnmp into async-mib-instrumentation

MIB instrumentation API changed to allow for asynchronous
managed objects access. Although built-in SNMPv2-SMI objects
are still synchronous, the MIB instrumentation API is async
what allows users to replace default MIB instrumentation
with their own, potentially asynchronous.

SMI/MIB managed objects API overhauled for simplicity and
flexibility breaking backward compatibility.

CHANGES.txt

@@ -5,9 +5,11 @@ Revision 5.0.0, released 2018-10-??
 - SNMPv3 crypto operations that require external dependencies
   made dependent on the optional external
   package -- pysnmpcrypto.
+
 - By switching to pysnmpcrypto, pysnmp effectively migrates from
   PyCryptodomex to pyca/cryptography whenever available on the
   platform.
+
 - Many really old backward-compatibility code snippets removed.
   Most importantly:
   
@@ -19,13 +21,20 @@ Revision 5.0.0, released 2018-10-??
 - The MIB instrumentation API overhauled in backward incompatible
   way:
 
-      - MIB instrumentation methods signatures simplified to accept
-        just var-binds (as var-arg), the rest of the parameters packed
-        into opaque kwargs
-      - CommandResponder application passes `snmpEngine` and optionally
-        user-supplied `cbCtx` object throughout the MIB instrumentation
-        methods. The goal is to let MIB objects access/modify whatever
-        custom Python objects they need while being called back.
+  * MIB instrumentation methods signatures simplified to accept
+    just var-binds (as var-arg), the rest of the parameters packed
+    into opaque kwargs
+
+  * CommandResponder application passes `snmpEngine` and optionally
+    user-supplied `cbCtx` object throughout the MIB instrumentation
+    methods. The goal is to let MIB objects access/modify whatever
+    custom Python objects they need while being called back.
+
+  * CommandResponder refactored to facilitate asynchronous
+    MIB instrumentation routines. The `readVars`, `readNextVars` and
+    `writeVars` MIB controller methods return immediately and
+    deliver their results via a call back.
+
 - The high-level API (`hlapi`) extended to cover lightweight SNMP v1arch
   in hope to ease the use of packet-level SNMP API.
 
@@ -41,6 +50,7 @@ Revision 5.0.0, released 2018-10-??
   automation around building well-formed SNMP messages is and mediating
   differences between SNMP versions is not present in this new `v1arch`
   layer.
+
 - The signature of the hlapi `.sendNotification()` call has changed
   to accept `*varBinds` instead of a sequence of `varBinds`. The rationale
   is to unify this method call with similar methods of CommandGenerator.

examples/smi/agent/custom-managed-object.py

@@ -51,13 +51,23 @@ if __name__ == '__main__':
 
     mibInstrum = instrum.MibInstrumController(mibBuilder)
 
+    def cbFun(varBinds, **context):
+        for oid, val in varBinds:
+            if exval.endOfMib.isSameTypeWith(val):
+                context['state']['stop'] = True
+            print('%s = %s' % ('.'.join([str(x) for x in oid]), not val.isValue and 'N/A' or val.prettyPrint()))
+
+        context['state']['varBinds'] = varBinds
+
+    context = {
+        'cbFun': cbFun,
+        'state': {
+            'varBinds': [((1, 3, 6), None)],
+            'stop': False
+        }
+    }
+
     print('Remote manager read access to MIB instrumentation (table walk)')
-
-    varBinds = [((), None)]
-
-    while True:
-        varBinds = mibInstrum.readNextVars(*varBinds)
-        oid, val = varBinds[0]
-        if exval.endOfMib.isSameTypeWith(val):
-            break
-        print(oid, val.prettyPrint())
+    while not context['state']['stop']:
+        mibInstrum.readNextVars(*context['state']['varBinds'], **context)
+    print('done')

examples/smi/agent/operations-on-managed-objects.py

@@ -26,24 +26,40 @@ snmpCommunityEntry, = mibBuilder.importSymbols(
 instanceId = snmpCommunityEntry.getInstIdFromIndices('my-router')
 print('done')
 
+
+def cbFun(varBinds, **context):
+    for oid, val in varBinds:
+        print('%s = %s' % ('.'.join([str(x) for x in oid]), not val.isValue and 'N/A' or val.prettyPrint()))
+
 print('Create/update SNMP-COMMUNITY-MIB::snmpCommunityEntry table row: ')
-varBinds = mibInstrum.writeVars(
+mibInstrum.writeVars(
     (snmpCommunityEntry.name + (2,) + instanceId, 'mycomm'),
     (snmpCommunityEntry.name + (3,) + instanceId, 'mynmsname'),
-    (snmpCommunityEntry.name + (7,) + instanceId, 'volatile')
+    (snmpCommunityEntry.name + (7,) + instanceId, 'volatile'),
+    cbFun=cbFun
 )
-for oid, val in varBinds:
-    print('%s = %s' % ('.'.join([str(x) for x in oid]), not val.isValue and 'N/A' or val.prettyPrint()))
 print('done')
 
+
+def cbFun(varBinds, **context):
+    for oid, val in varBinds:
+        if exval.endOfMib.isSameTypeWith(val):
+            context['state']['stop'] = True
+        print('%s = %s' % ('.'.join([str(x) for x in oid]), not val.isValue and 'N/A' or val.prettyPrint()))
+
+    context['state']['varBinds'] = varBinds
+
+context = {
+    'cbFun': cbFun,
+    'state': {
+        'varBinds': [((1, 3, 6), None)],
+        'stop': False
+    }
+}
+
 print('Read whole MIB (table walk)')
-varBinds = [((), None)]
-while True:
-    varBinds = mibInstrum.readNextVars(*varBinds)
-    oid, val = varBinds[0]
-    if exval.endOfMib.isSameTypeWith(val):
-        break
-    print('%s = %s' % ('.'.join([str(x) for x in oid]), not val.isValue and 'N/A' or val.prettyPrint()))
+while not context['state']['stop']:
+    mibInstrum.readNextVars(*context['state']['varBinds'], **context)
 print('done')
 
 print('Unloading MIB modules...'),

examples/v3arch/asyncore/agent/cmdrsp/custom-mib-controller.py

@@ -54,7 +54,9 @@ snmpContext = context.SnmpContext(snmpEngine)
 # always echos request var-binds in response.
 class EchoMibInstrumController(instrum.AbstractMibInstrumController):
     def readVars(self, *varBinds, **context):
-        return [(ov[0], v2c.OctetString('You queried OID %s' % ov[0])) for ov in varBinds]
+        cbFun = context.get('cbFun')
+        if cbFun:
+            cbFun([(ov[0], v2c.OctetString('You queried OID %s' % ov[0])) for ov in varBinds], **context)
 
 
 # Create a custom Management Instrumentation Controller and register at

pysnmp/entity/config.py

@@ -422,7 +422,8 @@ def delContext(snmpEngine, contextName):
     vacmContextEntry, tblIdx = __cookVacmContextInfo(snmpEngine, contextName)
 
     snmpEngine.msgAndPduDsp.mibInstrumController.writeVars(
-        (vacmContextEntry.name + (2,) + tblIdx, 'destroy')
+        (vacmContextEntry.name + (2,) + tblIdx, 'destroy'),
+        ** dict(snmpEngine=snmpEngine)
     )
 
 

pysnmp/entity/rfc3413/cmdrsp.py

@@ -25,8 +25,7 @@ class CommandResponderBase(object):
         self.cbCtx = cbCtx
         self.__pendingReqs = {}
 
-    def handleMgmtOperation(self, snmpEngine, stateReference,
-                            contextName, PDU, acCtx):
+    def initiateMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
         pass
 
     def close(self, snmpEngine):
@@ -147,7 +146,7 @@ class CommandResponderBase(object):
             'processPdu: stateReference %s, varBinds %s' % (stateReference, varBinds))
 
         try:
-            self.handleMgmtOperation(snmpEngine, stateReference, contextName, PDU)
+            self.initiateMgmtOperation(snmpEngine, stateReference, contextName, PDU)
 
         # SNMPv2 SMI exceptions
         except pysnmp.smi.error.GenError:
@@ -263,53 +262,79 @@ class CommandResponderBase(object):
 class GetCommandResponder(CommandResponderBase):
     pduTypes = (rfc1905.GetRequestPDU.tagSet,)
 
+    def completeMgmtOperation(self, varBinds, **context):
+        self.sendVarBinds(context['snmpEngine'], context['stateReference'],
+                          0, 0, varBinds)
+        self.releaseStateInformation(context['stateReference'])
+
     # rfc1905: 4.2.1
-    def handleMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
+    def initiateMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
         # rfc1905: 4.2.1.1
         mgmtFun = self.snmpContext.getMibInstrum(contextName).readVars
         varBinds = v2c.apiPDU.getVarBinds(PDU)
 
-        context = dict(snmpEngine=snmpEngine, acFun=self.verifyAccess, cbCtx=self.cbCtx)
+        context = dict(snmpEngine=snmpEngine,
+                       stateReference=stateReference,
+                       acFun=self.verifyAccess,
+                       cbFun=self.completeMgmtOperation,
+                       cbCtx=self.cbCtx)
 
-        rspVarBinds = mgmtFun(*varBinds, **context)
-
-        self.sendVarBinds(snmpEngine, stateReference, 0, 0, rspVarBinds)
-        self.releaseStateInformation(stateReference)
+        mgmtFun(*varBinds, **context)
 
 
 class NextCommandResponder(CommandResponderBase):
     pduTypes = (rfc1905.GetNextRequestPDU.tagSet,)
 
+    def completeMgmtOperation(self, varBinds, **context):
+        self.sendVarBinds(context['snmpEngine'], context['stateReference'],
+                          0, 0, varBinds)
+        self.releaseStateInformation(context['stateReference'])
+
     # rfc1905: 4.2.2
-    def handleMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
+    def initiateMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
         # rfc1905: 4.2.2.1
         mgmtFun = self.snmpContext.getMibInstrum(contextName).readNextVars
 
         varBinds = v2c.apiPDU.getVarBinds(PDU)
 
-        context = dict(snmpEngine=snmpEngine, acFun=self.verifyAccess, cbCtx=self.cbCtx)
+        context = dict(snmpEngine=snmpEngine,
+                       stateReference=stateReference,
+                       acFun=self.verifyAccess,
+                       cbFun=self.completeMgmtOperation,
+                       cbCtx=self.cbCtx)
 
-        while True:
-            rspVarBinds = mgmtFun(*varBinds, **context)
-
-            try:
-                self.sendVarBinds(snmpEngine, stateReference, 0, 0, rspVarBinds)
-
-            except error.StatusInformation:
-                idx = sys.exc_info()[1]['idx']
-                varBinds[idx] = (rspVarBinds[idx][0], varBinds[idx][1])
-            else:
-                break
-
-        self.releaseStateInformation(stateReference)
+        mgmtFun(*varBinds, **context)
 
 
 class BulkCommandResponder(CommandResponderBase):
     pduTypes = (rfc1905.GetBulkRequestPDU.tagSet,)
     maxVarBinds = 64
 
+    def _completeNonRepeaters(self, varBinds, **context):
+        context['rspVarBinds'][:] = varBinds
+        context['cbFun'] = self.completeMgmtOperation
+
+        mgmtFun = self.snmpContext.getMibInstrum(context['contextName']).readNextVars
+
+        mgmtFun(*context['varBinds'], **context)
+
+    def completeMgmtOperation(self, varBinds, **context):
+        context['rspVarBinds'].extend(varBinds)
+        context['counters']['M'] -= 1
+
+        if context['counters']['M'] and context['counters']['R']:
+            mgmtFun = self.snmpContext.getMibInstrum(context['contextName']).readNextVars
+
+            context['cbFun'] = self.completeMgmtOperation
+            mgmtFun(*varBinds[-context['counters']['R']:], **context)
+
+        else:
+            self.sendVarBinds(context['snmpEngine'], context['stateReference'],
+                              0, 0, varBinds)
+            self.releaseStateInformation(context['stateReference'])
+
     # rfc1905: 4.2.3
-    def handleMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
+    def initiateMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
         nonRepeaters = v2c.apiBulkPDU.getNonRepeaters(PDU)
         if nonRepeaters < 0:
             nonRepeaters = 0
@@ -318,68 +343,60 @@ class BulkCommandResponder(CommandResponderBase):
         if maxRepetitions < 0:
             maxRepetitions = 0
 
-        reqVarBinds = v2c.apiPDU.getVarBinds(PDU)
+        varBinds = v2c.apiPDU.getVarBinds(PDU)
 
-        N = min(int(nonRepeaters), len(reqVarBinds))
+        N = min(int(nonRepeaters), len(varBinds))
         M = int(maxRepetitions)
-        R = max(len(reqVarBinds) - N, 0)
+        R = max(len(varBinds) - N, 0)
 
         if R:
             M = min(M, self.maxVarBinds // R)
 
-        debug.logger & debug.flagApp and debug.logger('handleMgmtOperation: N %d, M %d, R %d' % (N, M, R))
+        debug.logger & debug.flagApp and debug.logger(
+            'initiateMgmtOperation: N %d, M %d, R %d' % (N, M, R))
 
         mgmtFun = self.snmpContext.getMibInstrum(contextName).readNextVars
 
-        context = dict(snmpEngine=snmpEngine, acFun=self.verifyAccess, cbCtx=self.cbCtx)
+        context = dict(snmpEngine=snmpEngine,
+                       stateReference=stateReference,
+                       contextName=contextName,
+                       acFun=self.verifyAccess,
+                       cbFun=self._completeNonRepeaters,
+                       cbCtx=self.cbCtx,
+                       varBinds=varBinds[-R:],
+                       counters={'M': M, 'R': R},
+                       rspVarBinds=[])
 
-        if N:
-            # TODO(etingof): manage all PDU var-binds in a single call
-            rspVarBinds = mgmtFun(*reqVarBinds[:N], **context)
-
-        else:
-            rspVarBinds = []
-
-        varBinds = reqVarBinds[-R:]
-
-        while M and R:
-            rspVarBinds.extend(mgmtFun(*varBinds, **context))
-            varBinds = rspVarBinds[-R:]
-            M -= 1
-
-        if len(rspVarBinds):
-            self.sendVarBinds(snmpEngine, stateReference, 0, 0, rspVarBinds)
-            self.releaseStateInformation(stateReference)
-        else:
-            raise pysnmp.smi.error.SmiError()
+        mgmtFun(*varBinds[:N], **context)
 
 
 class SetCommandResponder(CommandResponderBase):
     pduTypes = (rfc1905.SetRequestPDU.tagSet,)
 
+    def completeMgmtOperation(self, varBinds, **context):
+        self.sendVarBinds(context['snmpEngine'], context['stateReference'],
+                          0, 0, varBinds)
+        self.releaseStateInformation(context['stateReference'])
+
     # rfc1905: 4.2.5
-    def handleMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
+    def initiateMgmtOperation(self, snmpEngine, stateReference, contextName, PDU):
         mgmtFun = self.snmpContext.getMibInstrum(contextName).writeVars
 
         varBinds = v2c.apiPDU.getVarBinds(PDU)
 
-        instrumError = None
-
-        context = dict(snmpEngine=snmpEngine, acFun=self.verifyAccess, cbCtx=self.cbCtx)
+        context = dict(snmpEngine=snmpEngine,
+                       stateReference=stateReference,
+                       acFun=self.verifyAccess,
+                       cbFun=self.completeMgmtOperation,
+                       cbCtx=self.cbCtx)
 
         # rfc1905: 4.2.5.1-13
         try:
-            rspVarBinds = mgmtFun(*varBinds, **context)
+            mgmtFun(*varBinds, **context)
 
         except (pysnmp.smi.error.NoSuchObjectError,
                 pysnmp.smi.error.NoSuchInstanceError):
             instrumError = pysnmp.smi.error.NotWritableError()
             instrumError.update(sys.exc_info()[1])
-
-        else:
-            self.sendVarBinds(snmpEngine, stateReference, 0, 0, rspVarBinds)
-
-        self.releaseStateInformation(stateReference)
-
-        if instrumError:
+            self.releaseStateInformation(stateReference)
             raise instrumError

pysnmp/smi/instrum.py

@@ -184,15 +184,25 @@ class MibInstrumController(AbstractMibInstrumController):
     # MIB instrumentation
 
     def flipFlopFsm(self, fsmTable, *varBinds, **context):
-        self.__indexMib()
 
-        debug.logger & debug.flagIns and debug.logger('flipFlopFsm: input var-binds %r' % (varBinds,))
+        try:
+            fsmContext = context['fsmState']
+
+        except KeyError:
+            self.__indexMib()
+
+            fsmContext = context['fsmState'] = dict(varBinds=[], state='start', status='ok')
+
+            debug.logger & debug.flagIns and debug.logger('flipFlopFsm: input var-binds %r' % (varBinds,))
 
         mibTree, = self.mibBuilder.importSymbols('SNMPv2-SMI', 'iso')
 
-        outputVarBinds = []
-        state, status = 'start', 'ok'
+        outputVarBinds = fsmContext['varBinds']
+        state = fsmContext['state']
+        status = fsmContext['status']
+
         origExc = origTraceback = None
+
         while True:
             k = state, status
             if k in fsmTable:
@@ -237,7 +247,7 @@ class MibInstrumController(AbstractMibInstrumController):
                     break
                 else:
                     debug.logger & debug.flagIns and debug.logger(
-                        'flipFlopFsm: fun %s suceeded for %s=%r' % (mgmtFun, name, val))
+                        'flipFlopFsm: fun %s succeeded for %s=%r' % (mgmtFun, name, val))
                     if rval is not None:
                         outputVarBinds.append((rval[0], rval[1]))
 
@@ -252,13 +262,15 @@ class MibInstrumController(AbstractMibInstrumController):
                     # (seems to be irrelevant on Py3 but just in case)
                     del origTraceback
 
-        return outputVarBinds
+        cbFun = context.get('cbFun')
+        if cbFun:
+            cbFun(outputVarBinds, **context)
 
     def readVars(self, *varBinds, **context):
-        return self.flipFlopFsm(self.fsmReadVar, *varBinds, **context)
+        self.flipFlopFsm(self.fsmReadVar, *varBinds, **context)
 
     def readNextVars(self, *varBinds, **context):
-        return self.flipFlopFsm(self.fsmReadNextVar, *varBinds, **context)
+        self.flipFlopFsm(self.fsmReadNextVar, *varBinds, **context)
 
     def writeVars(self, *varBinds, **context):
-        return self.flipFlopFsm(self.fsmWriteVar, *varBinds, **context)
+        self.flipFlopFsm(self.fsmWriteVar, *varBinds, **context)