326 lines
12 KiB
Python
326 lines
12 KiB
Python
|
# Natural Language Toolkit: Transformation-based learning
|
||
|
|
#
|
||
|
|
# Copyright (C) 2001-2023 NLTK Project
|
||
|
|
# Author: Marcus Uneson <marcus.uneson@gmail.com>
|
||
|
|
# based on previous (nltk2) version by
|
||
|
|
# Christopher Maloof, Edward Loper, Steven Bird
|
||
|
|
# URL: <https://www.nltk.org/>
|
||
|
|
# For license information, see LICENSE.TXT
|
||
|
|
|
||
|
|
import itertools as it
|
||
|
|
from abc import ABCMeta, abstractmethod
|
||
|
|
|
||
|
|
from nltk.tbl.feature import Feature
|
||
|
|
from nltk.tbl.rule import Rule
|
||
|
|
|
||
|
|
|
||
|
|
class BrillTemplateI(metaclass=ABCMeta):
|
||
|
|
"""
|
||
|
|
An interface for generating lists of transformational rules that
|
||
|
|
apply at given sentence positions. ``BrillTemplateI`` is used by
|
||
|
|
``Brill`` training algorithms to generate candidate rules.
|
||
|
|
"""
|
||
|
|
|
||
|
|
@abstractmethod
|
||
|
|
def applicable_rules(self, tokens, i, correctTag):
|
||
|
|
"""
|
||
|
|
Return a list of the transformational rules that would correct
|
||
|
|
the ``i``-th subtoken's tag in the given token. In particular,
|
||
|
|
return a list of zero or more rules that would change
|
||
|
|
``tokens[i][1]`` to ``correctTag``, if applied to ``token[i]``.
|
||
|
|
|
||
|
|
If the ``i``-th token already has the correct tag (i.e., if
|
||
|
|
``tagged_tokens[i][1] == correctTag``), then
|
||
|
|
``applicable_rules()`` should return the empty list.
|
||
|
|
|
||
|
|
:param tokens: The tagged tokens being tagged.
|
||
|
|
:type tokens: list(tuple)
|
||
|
|
:param i: The index of the token whose tag should be corrected.
|
||
|
|
:type i: int
|
||
|
|
:param correctTag: The correct tag for the ``i``-th token.
|
||
|
|
:type correctTag: any
|
||
|
|
:rtype: list(BrillRule)
|
||
|
|
"""
|
||
|
|
|
||
|
|
@abstractmethod
|
||
|
|
def get_neighborhood(self, token, index):
|
||
|
|
"""
|
||
|
|
Returns the set of indices *i* such that
|
||
|
|
``applicable_rules(token, i, ...)`` depends on the value of
|
||
|
|
the *index*th token of *token*.
|
||
|
|
|
||
|
|
This method is used by the "fast" Brill tagger trainer.
|
||
|
|
|
||
|
|
:param token: The tokens being tagged.
|
||
|
|
:type token: list(tuple)
|
||
|
|
:param index: The index whose neighborhood should be returned.
|
||
|
|
:type index: int
|
||
|
|
:rtype: set
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
class Template(BrillTemplateI):
|
||
|
|
"""
|
||
|
|
A tbl Template that generates a list of L{Rule}s that apply at a given sentence
|
||
|
|
position. In particular, each C{Template} is parameterized by a list of
|
||
|
|
independent features (a combination of a specific
|
||
|
|
property to extract and a list C{L} of relative positions at which to extract
|
||
|
|
it) and generates all Rules that:
|
||
|
|
|
||
|
|
- use the given features, each at its own independent position; and
|
||
|
|
- are applicable to the given token.
|
||
|
|
"""
|
||
|
|
|
||
|
|
ALLTEMPLATES = []
|
||
|
|
# record a unique id of form "001", for each template created
|
||
|
|
# _ids = it.count(0)
|
||
|
|
|
||
|
|
def __init__(self, *features):
|
||
|
|
|
||
|
|
"""
|
||
|
|
Construct a Template for generating Rules.
|
||
|
|
|
||
|
|
Takes a list of Features. A C{Feature} is a combination
|
||
|
|
of a specific property and its relative positions and should be
|
||
|
|
a subclass of L{nltk.tbl.feature.Feature}.
|
||
|
|
|
||
|
|
An alternative calling convention (kept for backwards compatibility,
|
||
|
|
but less expressive as it only permits one feature type) is
|
||
|
|
Template(Feature, (start1, end1), (start2, end2), ...)
|
||
|
|
In new code, that would be better written
|
||
|
|
Template(Feature(start1, end1), Feature(start2, end2), ...)
|
||
|
|
|
||
|
|
For instance, importing some features
|
||
|
|
|
||
|
|
>>> from nltk.tbl.template import Template
|
||
|
|
>>> from nltk.tag.brill import Word, Pos
|
||
|
|
|
||
|
|
Create some features
|
||
|
|
|
||
|
|
>>> wfeat1, wfeat2, pfeat = (Word([-1]), Word([1,2]), Pos([-2,-1]))
|
||
|
|
|
||
|
|
Create a single-feature template
|
||
|
|
|
||
|
|
>>> Template(wfeat1)
|
||
|
|
Template(Word([-1]))
|
||
|
|
|
||
|
|
Or a two-feature one
|
||
|
|
|
||
|
|
>>> Template(wfeat1, wfeat2)
|
||
|
|
Template(Word([-1]),Word([1, 2]))
|
||
|
|
|
||
|
|
Or a three-feature one with two different feature types
|
||
|
|
|
||
|
|
>>> Template(wfeat1, wfeat2, pfeat)
|
||
|
|
Template(Word([-1]),Word([1, 2]),Pos([-2, -1]))
|
||
|
|
|
||
|
|
deprecated api: Feature subclass, followed by list of (start,end) pairs
|
||
|
|
(permits only a single Feature)
|
||
|
|
|
||
|
|
>>> Template(Word, (-2,-1), (0,0))
|
||
|
|
Template(Word([-2, -1]),Word([0]))
|
||
|
|
|
||
|
|
Incorrect specification raises TypeError
|
||
|
|
|
||
|
|
>>> Template(Word, (-2,-1), Pos, (0,0))
|
||
|
|
Traceback (most recent call last):
|
||
|
|
File "<stdin>", line 1, in <module>
|
||
|
|
File "nltk/tag/tbl/template.py", line 143, in __init__
|
||
|
|
raise TypeError(
|
||
|
|
TypeError: expected either Feature1(args), Feature2(args), ... or Feature, (start1, end1), (start2, end2), ...
|
||
|
|
|
||
|
|
:type features: list of Features
|
||
|
|
:param features: the features to build this Template on
|
||
|
|
"""
|
||
|
|
# determine the calling form: either
|
||
|
|
# Template(Feature, args1, [args2, ...)]
|
||
|
|
# Template(Feature1(args), Feature2(args), ...)
|
||
|
|
if all(isinstance(f, Feature) for f in features):
|
||
|
|
self._features = features
|
||
|
|
elif issubclass(features[0], Feature) and all(
|
||
|
|
isinstance(a, tuple) for a in features[1:]
|
||
|
|
):
|
||
|
|
self._features = [features[0](*tp) for tp in features[1:]]
|
||
|
|
else:
|
||
|
|
raise TypeError(
|
||
|
|
"expected either Feature1(args), Feature2(args), ... or Feature, (start1, end1), (start2, end2), ..."
|
||
|
|
)
|
||
|
|
self.id = f"{len(self.ALLTEMPLATES):03d}"
|
||
|
|
self.ALLTEMPLATES.append(self)
|
||
|
|
|
||
|
|
def __repr__(self):
|
||
|
|
return "{}({})".format(
|
||
|
|
self.__class__.__name__,
|
||
|
|
",".join([str(f) for f in self._features]),
|
||
|
|
)
|
||
|
|
|
||
|
|
def applicable_rules(self, tokens, index, correct_tag):
|
||
|
|
if tokens[index][1] == correct_tag:
|
||
|
|
return []
|
||
|
|
|
||
|
|
# For each of this Template's features, find the conditions
|
||
|
|
# that are applicable for the given token.
|
||
|
|
# Then, generate one Rule for each combination of features
|
||
|
|
# (the crossproduct of the conditions).
|
||
|
|
|
||
|
|
applicable_conditions = self._applicable_conditions(tokens, index)
|
||
|
|
xs = list(it.product(*applicable_conditions))
|
||
|
|
return [Rule(self.id, tokens[index][1], correct_tag, tuple(x)) for x in xs]
|
||
|
|
|
||
|
|
def _applicable_conditions(self, tokens, index):
|
||
|
|
"""
|
||
|
|
:returns: A set of all conditions for rules
|
||
|
|
that are applicable to C{tokens[index]}.
|
||
|
|
"""
|
||
|
|
conditions = []
|
||
|
|
|
||
|
|
for feature in self._features:
|
||
|
|
conditions.append([])
|
||
|
|
for pos in feature.positions:
|
||
|
|
if not (0 <= index + pos < len(tokens)):
|
||
|
|
continue
|
||
|
|
value = feature.extract_property(tokens, index + pos)
|
||
|
|
conditions[-1].append((feature, value))
|
||
|
|
return conditions
|
||
|
|
|
||
|
|
def get_neighborhood(self, tokens, index):
|
||
|
|
# inherit docs from BrillTemplateI
|
||
|
|
|
||
|
|
# applicable_rules(tokens, index, ...) depends on index.
|
||
|
|
neighborhood = {index} # set literal for python 2.7+
|
||
|
|
|
||
|
|
# applicable_rules(tokens, i, ...) depends on index if
|
||
|
|
# i+start < index <= i+end.
|
||
|
|
|
||
|
|
allpositions = [0] + [p for feat in self._features for p in feat.positions]
|
||
|
|
start, end = min(allpositions), max(allpositions)
|
||
|
|
s = max(0, index + (-end))
|
||
|
|
e = min(index + (-start) + 1, len(tokens))
|
||
|
|
for i in range(s, e):
|
||
|
|
neighborhood.add(i)
|
||
|
|
return neighborhood
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def expand(cls, featurelists, combinations=None, skipintersecting=True):
|
||
|
|
|
||
|
|
"""
|
||
|
|
Factory method to mass generate Templates from a list L of lists of Features.
|
||
|
|
|
||
|
|
#With combinations=(k1, k2), the function will in all possible ways choose k1 ... k2
|
||
|
|
#of the sublists in L; it will output all Templates formed by the Cartesian product
|
||
|
|
#of this selection, with duplicates and other semantically equivalent
|
||
|
|
#forms removed. Default for combinations is (1, len(L)).
|
||
|
|
|
||
|
|
The feature lists may have been specified
|
||
|
|
manually, or generated from Feature.expand(). For instance,
|
||
|
|
|
||
|
|
>>> from nltk.tbl.template import Template
|
||
|
|
>>> from nltk.tag.brill import Word, Pos
|
||
|
|
|
||
|
|
#creating some features
|
||
|
|
>>> (wd_0, wd_01) = (Word([0]), Word([0,1]))
|
||
|
|
|
||
|
|
>>> (pos_m2, pos_m33) = (Pos([-2]), Pos([3-2,-1,0,1,2,3]))
|
||
|
|
|
||
|
|
>>> list(Template.expand([[wd_0], [pos_m2]]))
|
||
|
|
[Template(Word([0])), Template(Pos([-2])), Template(Pos([-2]),Word([0]))]
|
||
|
|
|
||
|
|
>>> list(Template.expand([[wd_0, wd_01], [pos_m2]]))
|
||
|
|
[Template(Word([0])), Template(Word([0, 1])), Template(Pos([-2])), Template(Pos([-2]),Word([0])), Template(Pos([-2]),Word([0, 1]))]
|
||
|
|
|
||
|
|
#note: with Feature.expand(), it is very easy to generate more templates
|
||
|
|
#than your system can handle -- for instance,
|
||
|
|
>>> wordtpls = Word.expand([-2,-1,0,1], [1,2], excludezero=False)
|
||
|
|
>>> len(wordtpls)
|
||
|
|
7
|
||
|
|
|
||
|
|
>>> postpls = Pos.expand([-3,-2,-1,0,1,2], [1,2,3], excludezero=True)
|
||
|
|
>>> len(postpls)
|
||
|
|
9
|
||
|
|
|
||
|
|
#and now the Cartesian product of all non-empty combinations of two wordtpls and
|
||
|
|
#two postpls, with semantic equivalents removed
|
||
|
|
>>> templates = list(Template.expand([wordtpls, wordtpls, postpls, postpls]))
|
||
|
|
>>> len(templates)
|
||
|
|
713
|
||
|
|
|
||
|
|
|
||
|
|
will return a list of eight templates
|
||
|
|
Template(Word([0])),
|
||
|
|
Template(Word([0, 1])),
|
||
|
|
Template(Pos([-2])),
|
||
|
|
Template(Pos([-1])),
|
||
|
|
Template(Pos([-2]),Word([0])),
|
||
|
|
Template(Pos([-1]),Word([0])),
|
||
|
|
Template(Pos([-2]),Word([0, 1])),
|
||
|
|
Template(Pos([-1]),Word([0, 1]))]
|
||
|
|
|
||
|
|
|
||
|
|
#Templates where one feature is a subset of another, such as
|
||
|
|
#Template(Word([0,1]), Word([1]), will not appear in the output.
|
||
|
|
#By default, this non-subset constraint is tightened to disjointness:
|
||
|
|
#Templates of type Template(Word([0,1]), Word([1,2]) will also be filtered out.
|
||
|
|
#With skipintersecting=False, then such Templates are allowed
|
||
|
|
|
||
|
|
WARNING: this method makes it very easy to fill all your memory when training
|
||
|
|
generated templates on any real-world corpus
|
||
|
|
|
||
|
|
:param featurelists: lists of Features, whose Cartesian product will return a set of Templates
|
||
|
|
:type featurelists: list of (list of Features)
|
||
|
|
:param combinations: given n featurelists: if combinations=k, all generated Templates will have
|
||
|
|
k features; if combinations=(k1,k2) they will have k1..k2 features; if None, defaults to 1..n
|
||
|
|
:type combinations: None, int, or (int, int)
|
||
|
|
:param skipintersecting: if True, do not output intersecting Templates (non-disjoint positions for some feature)
|
||
|
|
:type skipintersecting: bool
|
||
|
|
:returns: generator of Templates
|
||
|
|
|
||
|
|
"""
|
||
|
|
|
||
|
|
def nonempty_powerset(xs): # xs is a list
|
||
|
|
# itertools docnonempty_powerset([1,2,3]) --> (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)
|
||
|
|
|
||
|
|
# find the correct tuple given combinations, one of {None, k, (k1,k2)}
|
||
|
|
k = combinations # for brevity
|
||
|
|
combrange = (
|
||
|
|
(1, len(xs) + 1)
|
||
|
|
if k is None
|
||
|
|
else (k, k + 1) # n over 1 .. n over n (all non-empty combinations)
|
||
|
|
if isinstance(k, int)
|
||
|
|
else (k[0], k[1] + 1) # n over k (only
|
||
|
|
) # n over k1, n over k1+1... n over k2
|
||
|
|
return it.chain.from_iterable(
|
||
|
|
it.combinations(xs, r) for r in range(*combrange)
|
||
|
|
)
|
||
|
|
|
||
|
|
seentemplates = set()
|
||
|
|
for picks in nonempty_powerset(featurelists):
|
||
|
|
for pick in it.product(*picks):
|
||
|
|
if any(
|
||
|
|
i != j and x.issuperset(y)
|
||
|
|
for (i, x) in enumerate(pick)
|
||
|
|
for (j, y) in enumerate(pick)
|
||
|
|
):
|
||
|
|
continue
|
||
|
|
if skipintersecting and any(
|
||
|
|
i != j and x.intersects(y)
|
||
|
|
for (i, x) in enumerate(pick)
|
||
|
|
for (j, y) in enumerate(pick)
|
||
|
|
):
|
||
|
|
continue
|
||
|
|
thistemplate = cls(*sorted(pick))
|
||
|
|
strpick = str(thistemplate)
|
||
|
|
#!!FIXME --this is hackish
|
||
|
|
if strpick in seentemplates: # already added
|
||
|
|
cls._poptemplate()
|
||
|
|
continue
|
||
|
|
seentemplates.add(strpick)
|
||
|
|
yield thistemplate
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def _cleartemplates(cls):
|
||
|
|
cls.ALLTEMPLATES = []
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def _poptemplate(cls):
|
||
|
|
return cls.ALLTEMPLATES.pop() if cls.ALLTEMPLATES else None
|