96 lines
3.5 KiB
Python
96 lines
3.5 KiB
Python
|
import unittest
|
||
|
|
|
||
|
|
from nltk.corpus import brown
|
||
|
|
from nltk.jsontags import JSONTaggedDecoder, JSONTaggedEncoder
|
||
|
|
from nltk.tag import (
|
||
|
|
AffixTagger,
|
||
|
|
BigramTagger,
|
||
|
|
BrillTagger,
|
||
|
|
BrillTaggerTrainer,
|
||
|
|
DefaultTagger,
|
||
|
|
NgramTagger,
|
||
|
|
PerceptronTagger,
|
||
|
|
RegexpTagger,
|
||
|
|
TrigramTagger,
|
||
|
|
UnigramTagger,
|
||
|
|
)
|
||
|
|
from nltk.tag.brill import nltkdemo18
|
||
|
|
|
||
|
|
|
||
|
|
class TestJSONSerialization(unittest.TestCase):
|
||
|
|
def setUp(self):
|
||
|
|
self.corpus = brown.tagged_sents()[:35]
|
||
|
|
self.decoder = JSONTaggedDecoder()
|
||
|
|
self.encoder = JSONTaggedEncoder()
|
||
|
|
self.default_tagger = DefaultTagger("NN")
|
||
|
|
|
||
|
|
def test_default_tagger(self):
|
||
|
|
encoded = self.encoder.encode(self.default_tagger)
|
||
|
|
decoded = self.decoder.decode(encoded)
|
||
|
|
|
||
|
|
self.assertEqual(repr(self.default_tagger), repr(decoded))
|
||
|
|
self.assertEqual(self.default_tagger._tag, decoded._tag)
|
||
|
|
|
||
|
|
def test_regexp_tagger(self):
|
||
|
|
tagger = RegexpTagger([(r".*", "NN")], backoff=self.default_tagger)
|
||
|
|
|
||
|
|
encoded = self.encoder.encode(tagger)
|
||
|
|
decoded = self.decoder.decode(encoded)
|
||
|
|
|
||
|
|
self.assertEqual(repr(tagger), repr(decoded))
|
||
|
|
self.assertEqual(repr(tagger.backoff), repr(decoded.backoff))
|
||
|
|
self.assertEqual(tagger._regexps, decoded._regexps)
|
||
|
|
|
||
|
|
def test_affix_tagger(self):
|
||
|
|
tagger = AffixTagger(self.corpus, backoff=self.default_tagger)
|
||
|
|
|
||
|
|
encoded = self.encoder.encode(tagger)
|
||
|
|
decoded = self.decoder.decode(encoded)
|
||
|
|
|
||
|
|
self.assertEqual(repr(tagger), repr(decoded))
|
||
|
|
self.assertEqual(repr(tagger.backoff), repr(decoded.backoff))
|
||
|
|
self.assertEqual(tagger._affix_length, decoded._affix_length)
|
||
|
|
self.assertEqual(tagger._min_word_length, decoded._min_word_length)
|
||
|
|
self.assertEqual(tagger._context_to_tag, decoded._context_to_tag)
|
||
|
|
|
||
|
|
def test_ngram_taggers(self):
|
||
|
|
unitagger = UnigramTagger(self.corpus, backoff=self.default_tagger)
|
||
|
|
bitagger = BigramTagger(self.corpus, backoff=unitagger)
|
||
|
|
tritagger = TrigramTagger(self.corpus, backoff=bitagger)
|
||
|
|
ntagger = NgramTagger(4, self.corpus, backoff=tritagger)
|
||
|
|
|
||
|
|
encoded = self.encoder.encode(ntagger)
|
||
|
|
decoded = self.decoder.decode(encoded)
|
||
|
|
|
||
|
|
self.assertEqual(repr(ntagger), repr(decoded))
|
||
|
|
self.assertEqual(repr(tritagger), repr(decoded.backoff))
|
||
|
|
self.assertEqual(repr(bitagger), repr(decoded.backoff.backoff))
|
||
|
|
self.assertEqual(repr(unitagger), repr(decoded.backoff.backoff.backoff))
|
||
|
|
self.assertEqual(
|
||
|
|
repr(self.default_tagger), repr(decoded.backoff.backoff.backoff.backoff)
|
||
|
|
)
|
||
|
|
|
||
|
|
def test_perceptron_tagger(self):
|
||
|
|
tagger = PerceptronTagger(load=False)
|
||
|
|
tagger.train(self.corpus)
|
||
|
|
|
||
|
|
encoded = self.encoder.encode(tagger)
|
||
|
|
decoded = self.decoder.decode(encoded)
|
||
|
|
|
||
|
|
self.assertEqual(tagger.model.weights, decoded.model.weights)
|
||
|
|
self.assertEqual(tagger.tagdict, decoded.tagdict)
|
||
|
|
self.assertEqual(tagger.classes, decoded.classes)
|
||
|
|
|
||
|
|
def test_brill_tagger(self):
|
||
|
|
trainer = BrillTaggerTrainer(
|
||
|
|
self.default_tagger, nltkdemo18(), deterministic=True
|
||
|
|
)
|
||
|
|
tagger = trainer.train(self.corpus, max_rules=30)
|
||
|
|
|
||
|
|
encoded = self.encoder.encode(tagger)
|
||
|
|
decoded = self.decoder.decode(encoded)
|
||
|
|
|
||
|
|
self.assertEqual(repr(tagger._initial_tagger), repr(decoded._initial_tagger))
|
||
|
|
self.assertEqual(tagger._rules, decoded._rules)
|
||
|
|
self.assertEqual(tagger._training_stats, decoded._training_stats)
|