22 lines
413 B
Python
22 lines
413 B
Python
|
|
||
|
|
||
|
class StreamTokenizer:
|
||
|
|
||
|
WHITESPACE = [' ','\n','\r','\t','\d']
|
||
|
|
||
|
def __init__(self,stream):
|
||
|
self.__stream = stream
|
||
|
|
||
|
def read(self,len = 1):
|
||
|
self.__stream.read(len)
|
||
|
|
||
|
def readToSeparator(self,separator = WHITESPACE):
|
||
|
token = bytearray()
|
||
|
c = self.__stream.read(1)
|
||
|
while (not c is None) and (len(c) != 0) and (not c in separator):
|
||
|
token.append(c)
|
||
|
c = self.__stream.read(1)
|
||
|
|
||
|
return token
|
||
|
|