22 lines
413 B
Python
22 lines
413 B
Python
|
|
|
|
class StreamTokenizer:
|
|
|
|
WHITESPACE = [' ','\n','\r','\t','\d']
|
|
|
|
def __init__(self,stream):
|
|
self.__stream = stream
|
|
|
|
def read(self,len = 1):
|
|
self.__stream.read(len)
|
|
|
|
def readToSeparator(self,separator = WHITESPACE):
|
|
token = bytearray()
|
|
c = self.__stream.read(1)
|
|
while (not c is None) and (len(c) != 0) and (not c in separator):
|
|
token.append(c)
|
|
c = self.__stream.read(1)
|
|
|
|
return token
|
|
|