92 lines
3.8 KiB
Python
92 lines
3.8 KiB
Python
# -*- coding: utf-8 -*-
|
||
import re
|
||
from pysbd.utils import Rule
|
||
|
||
class Common(object):
|
||
|
||
# added special case: r"[。．.！!? ]{2,}" to handle intermittent dots, exclamation, etc.
|
||
# r"[。．.！!?] at end to handle single instances of these symbol inputs
|
||
SENTENCE_BOUNDARY_REGEX = r"（(?:[^）])*）(?=\s?[A-Z])|「(?:[^」])*」(?=\s[A-Z])|\((?:[^\)]){2,}\)(?=\s[A-Z])|\'(?:[^\'])*[^,]\'(?=\s[A-Z])|\"(?:[^\"])*[^,]\"(?=\s[A-Z])|\“(?:[^\”])*[^,]\”(?=\s[A-Z])|[。．.！!?？ ]{2,}|\S.*?[。．.！!?？ȸȹ☉☈☇☄]|[。．.！!?？]"
|
||
|
||
# # Rubular: http://rubular.com/r/NqCqv372Ix
|
||
QUOTATION_AT_END_OF_SENTENCE_REGEX = r'[!?\.-][\"\'“”]\s{1}[A-Z]'
|
||
|
||
# # Rubular: http://rubular.com/r/6flGnUMEVl
|
||
PARENS_BETWEEN_DOUBLE_QUOTES_REGEX = r'["\”]\s\(.*\)\s["\“]'
|
||
|
||
# # Rubular: http://rubular.com/r/TYzr4qOW1Q
|
||
# BETWEEN_DOUBLE_QUOTES_REGEX = / "(?:[^"])*[^, ]"|“(?: [ ^”])*[^, ]”/
|
||
|
||
# # Rubular: http://rubular.com/r/JMjlZHAT4g
|
||
SPLIT_SPACE_QUOTATION_AT_END_OF_SENTENCE_REGEX = r'(?<=[!?\.-][\"\'“”])\s{1}(?=[A-Z])'
|
||
|
||
# # Rubular: http://rubular.com/r/mQ8Es9bxtk
|
||
CONTINUOUS_PUNCTUATION_REGEX = r'(?<=\S)(!|\?){3,}(?=(\s|\Z|$))'
|
||
|
||
# https://rubular.com/r/UkumQaILKbkeyc
|
||
# https://github.com/diasks2/pragmatic_segmenter/commit/d9ec1a352aff92b91e2e572c30bb9561eb42c703
|
||
NUMBERED_REFERENCE_REGEX = r'(?<=[^\d\s])(\.|∯)((\[(\d{1,3},?\s?-?\s?)*\b\d{1,3}\])+|((\d{1,3}\s?)?\d{1,3}))(\s)(?=[A-Z])'
|
||
|
||
# # Rubular: http://rubular.com/r/yqa4Rit8EY
|
||
PossessiveAbbreviationRule = Rule(r"\.(?='s\s)|\.(?='s$)|\.(?='s\Z)", '∯')
|
||
|
||
# # Rubular: http://rubular.com/r/NEv265G2X2
|
||
KommanditgesellschaftRule = Rule(r'(?<=Co)\.(?=\sKG)', '∯')
|
||
|
||
# # Rubular: http://rubular.com/r/xDkpFZ0EgH
|
||
MULTI_PERIOD_ABBREVIATION_REGEX = r"\b[a-z](?:\.[a-z])+[.]"
|
||
|
||
class SingleLetterAbbreviationRules(object):
|
||
"""Searches for periods within an abbreviation and
|
||
replaces the periods.
|
||
"""
|
||
# Rubular: http://rubular.com/r/e3H6kwnr6H
|
||
SingleUpperCaseLetterAtStartOfLineRule = Rule(r"(?<=^[A-Z])\.(?=\s)", '∯')
|
||
|
||
# Rubular: http://rubular.com/r/gitvf0YWH4
|
||
SingleUpperCaseLetterRule = Rule(r"(?<=\s[A-Z])\.(?=,?\s)", '∯')
|
||
|
||
All = [
|
||
SingleUpperCaseLetterAtStartOfLineRule, SingleUpperCaseLetterRule
|
||
]
|
||
|
||
class AmPmRules(object):
|
||
|
||
# Rubular: http://rubular.com/r/Vnx3m4Spc8
|
||
UpperCasePmRule = Rule(r'(?<= P∯M)∯(?=\s[A-Z])', '.')
|
||
|
||
# Rubular: http://rubular.com/r/AJMCotJVbW
|
||
UpperCaseAmRule = Rule(r'(?<=A∯M)∯(?=\s[A-Z])', '.')
|
||
|
||
# Rubular: http://rubular.com/r/13q7SnOhgA
|
||
LowerCasePmRule = Rule(r'(?<=p∯m)∯(?=\s[A-Z])', '.')
|
||
|
||
# Rubular: http://rubular.com/r/DgUDq4mLz5
|
||
LowerCaseAmRule = Rule(r'(?<=a∯m)∯(?=\s[A-Z])', '.')
|
||
|
||
All = [UpperCasePmRule, UpperCaseAmRule, LowerCasePmRule, LowerCaseAmRule]
|
||
|
||
class Numbers(object):
|
||
# Rubular: http://rubular.com/r/oNyxBOqbyy
|
||
PeriodBeforeNumberRule = Rule(r'\.(?=\d)', '∯')
|
||
|
||
# Rubular: http://rubular.com/r/EMk5MpiUzt
|
||
NumberAfterPeriodBeforeLetterRule = Rule(r'(?<=\d)\.(?=\S)', '∯')
|
||
|
||
# Rubular: http://rubular.com/r/rf4l1HjtjG
|
||
NewLineNumberPeriodSpaceLetterRule = Rule(r'(?<=\r\d)\.(?=(\s\S)|\))', '∯')
|
||
|
||
# Rubular: http://rubular.com/r/HPa4sdc6b9
|
||
StartLineNumberPeriodRule = Rule(r'(?<=^\d)\.(?=(\s\S)|\))', '∯')
|
||
|
||
# Rubular: http://rubular.com/r/NuvWnKleFl
|
||
StartLineTwoDigitNumberPeriodRule = Rule(r'(?<=^\d\d)\.(?=(\s\S)|\))', '∯')
|
||
|
||
All = [
|
||
PeriodBeforeNumberRule,
|
||
NumberAfterPeriodBeforeLetterRule,
|
||
NewLineNumberPeriodSpaceLetterRule,
|
||
StartLineNumberPeriodRule,
|
||
StartLineTwoDigitNumberPeriodRule
|
||
]
|