137 lines
4.7 KiB
Python
137 lines
4.7 KiB
Python
from __future__ import unicode_literals
|
|
|
|
import unittest
|
|
from io import StringIO
|
|
import string
|
|
|
|
from .. import Scanning
|
|
from ..Symtab import ModuleScope
|
|
from ..TreeFragment import StringParseContext
|
|
from ..Errors import init_thread
|
|
|
|
# generate some fake code - just a bunch of lines of the form "a0 a1 ..."
|
|
code = []
|
|
for ch in string.ascii_lowercase:
|
|
line = " ".join(["%s%s" % (ch, n) for n in range(10)])
|
|
code.append(line)
|
|
code = "\n".join(code)
|
|
|
|
init_thread()
|
|
|
|
|
|
class TestScanning(unittest.TestCase):
|
|
def make_scanner(self):
|
|
source = Scanning.StringSourceDescriptor("fake code", code)
|
|
buf = StringIO(code)
|
|
context = StringParseContext("fake context")
|
|
scope = ModuleScope("fake_module", None, None)
|
|
|
|
return Scanning.PyrexScanner(buf, source, scope=scope, context=context)
|
|
|
|
def test_put_back_positions(self):
|
|
scanner = self.make_scanner()
|
|
|
|
self.assertEqual(scanner.sy, "IDENT")
|
|
self.assertEqual(scanner.systring, "a0")
|
|
scanner.next()
|
|
self.assertEqual(scanner.sy, "IDENT")
|
|
self.assertEqual(scanner.systring, "a1")
|
|
a1pos = scanner.position()
|
|
self.assertEqual(a1pos[1:], (1, 3))
|
|
a2peek = scanner.peek() # shouldn't mess up the position
|
|
self.assertEqual(a1pos, scanner.position())
|
|
scanner.next()
|
|
self.assertEqual(a2peek, (scanner.sy, scanner.systring))
|
|
|
|
# find next line
|
|
while scanner.sy != "NEWLINE":
|
|
scanner.next()
|
|
|
|
line_sy = []
|
|
line_systring = []
|
|
line_pos = []
|
|
|
|
scanner.next()
|
|
while scanner.sy != "NEWLINE":
|
|
line_sy.append(scanner.sy)
|
|
line_systring.append(scanner.systring)
|
|
line_pos.append(scanner.position())
|
|
scanner.next()
|
|
|
|
for sy, systring, pos in zip(
|
|
line_sy[::-1], line_systring[::-1], line_pos[::-1]
|
|
):
|
|
scanner.put_back(sy, systring, pos)
|
|
|
|
n = 0
|
|
while scanner.sy != "NEWLINE":
|
|
self.assertEqual(scanner.sy, line_sy[n])
|
|
self.assertEqual(scanner.systring, line_systring[n])
|
|
self.assertEqual(scanner.position(), line_pos[n])
|
|
scanner.next()
|
|
n += 1
|
|
|
|
self.assertEqual(n, len(line_pos))
|
|
|
|
def test_tentatively_scan(self):
|
|
scanner = self.make_scanner()
|
|
with Scanning.tentatively_scan(scanner) as errors:
|
|
while scanner.sy != "NEWLINE":
|
|
scanner.next()
|
|
self.assertFalse(errors)
|
|
|
|
scanner.next()
|
|
self.assertEqual(scanner.systring, "b0")
|
|
pos = scanner.position()
|
|
with Scanning.tentatively_scan(scanner) as errors:
|
|
while scanner.sy != "NEWLINE":
|
|
scanner.next()
|
|
if scanner.systring == "b7":
|
|
scanner.error("Oh no not b7!")
|
|
break
|
|
self.assertTrue(errors)
|
|
self.assertEqual(scanner.systring, "b0") # state has been restored
|
|
self.assertEqual(scanner.position(), pos)
|
|
scanner.next()
|
|
self.assertEqual(scanner.systring, "b1") # and we can keep going again
|
|
scanner.next()
|
|
self.assertEqual(scanner.systring, "b2") # and we can keep going again
|
|
|
|
with Scanning.tentatively_scan(scanner) as error:
|
|
scanner.error("Something has gone wrong with the current symbol")
|
|
self.assertEqual(scanner.systring, "b2")
|
|
scanner.next()
|
|
self.assertEqual(scanner.systring, "b3")
|
|
|
|
# test a few combinations of nested scanning
|
|
sy1, systring1 = scanner.sy, scanner.systring
|
|
pos1 = scanner.position()
|
|
with Scanning.tentatively_scan(scanner):
|
|
scanner.next()
|
|
sy2, systring2 = scanner.sy, scanner.systring
|
|
pos2 = scanner.position()
|
|
with Scanning.tentatively_scan(scanner):
|
|
with Scanning.tentatively_scan(scanner):
|
|
scanner.next()
|
|
scanner.next()
|
|
scanner.error("Ooops")
|
|
self.assertEqual((scanner.sy, scanner.systring), (sy2, systring2))
|
|
self.assertEqual((scanner.sy, scanner.systring), (sy2, systring2))
|
|
scanner.error("eee")
|
|
self.assertEqual((scanner.sy, scanner.systring), (sy1, systring1))
|
|
with Scanning.tentatively_scan(scanner):
|
|
scanner.next()
|
|
scanner.next()
|
|
with Scanning.tentatively_scan(scanner):
|
|
scanner.next()
|
|
# no error - but this block should be unwound by the outer block too
|
|
scanner.next()
|
|
scanner.error("Oooops")
|
|
self.assertEqual((scanner.sy, scanner.systring), (sy1, systring1))
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|