英文稿分析-1

#!/usr/bin/env python3# -*- coding: utf-8 -*-import sysimport codecsimport stringimport nltkimport numpyimport mathimport igraphfrom matplotlib import pyplot as plt from matplotlib.font_manager import FontProperties class TextFileProcessing: __numList__ = None __puncList__ = None __stopWords__ = None __additional__ = ['also', 'paper', 'always', 'figure', 'table', 'shown', 'illustrated', 'shows'] def __init__(self): self.__numList__ = string.digits self.__puncList__ = string.punctuation self.__stopWords__ = set(nltk.corpus.stopwords.words('english')) def __filterOut__(self, word): _word = word if _word in self.__additional__: _word = '' for ch in self.__numList__: _word = _word.replace(ch, '') for ch in self.__puncList__: _word = _word.replace(ch, '') if _word in self.__stopWords__: _word = '' return _word def __isWord__(self, word): return len(nltk.corpus.wordnet.synsets(word)) > 0 def LoadFile(self, fileName): content = [] termFreqList = {} with codecs.open(fileName, 'r', 'utf-8') as textFifle: for line in textFifle: _line = line.strip().lower() _wordList = nltk.word_tokenize(_line) for word in _wordList: _word = self.__filterOut__(word) if len(_word) > 3 and self.__isWord__(_word): content.append(_word) if word in termFreqList: termFreqList[_word] = termFreqList[_word] + 1 else: termFreqList[_word] = 1 return (content, termFreqList) def LoadFile2Distance(self, fileName): content = [] with codecs.open(fileName, 'r', 'utf-8') as textFifle: for line in textFifle: _line = line.strip().lower() _wordList = nltk.word_tokenize(_line) for word in _wordList: _word = self.__filterOut__(word) if len(_word) > 3 and self.__isWord__(_word): content.append(_word) termFreqList = {} for pos in range(0, len(content) - 1): for w in range(pos + 1, len(content)): if content[pos] == content[w]: continue _word = '%s|%s' % (content[pos], content[w]) if content[pos] > content[w]: _word = '%s|%s' % (content[w], content[pos]) if _word in termFreqList: termFreqList[_word] += w - pos else: termFreqList[_word] = w - pos return (content, termFreqList) def LoadFile2Graph(self, fileName, minimal=30): content = [] with codecs.open(fileName, 'r', 'utf-8') as textFifle: for line in textFifle: _line = line.strip().lower() _wordList = nltk.word_tokenize(_line) for word in _wordList: _word = self.__filterOut__(word) if len(_word) > 3 and self.__isWord__(_word): content.append(_word) termFreqList = {} for pos in range(0, len(content) - 1): for w in range(pos + 1, len(content)): if content[pos] == content[w]: continue _word = '%s|%s' % (content[pos], content[w]) if content[pos] > content[w]: _word = '%s|%s' % (content[w], content[pos]) if _word in termFreqList: termFreqList[_word] += w - pos else: termFreqList[_word] = w - pos g = igraph.Graph() nodeLabels = [] for term in sorted(termFreqList, key=termFreqList.get, reverse=True): if minimal > 0: nodes = term.split('|') for node in nodes: try: _node = g.vs.find(name=node) except: g.add_vertex(node) nodeLabels.append(node) g.add_edge(nodes[0], nodes[1], weight=termFreqList[term]) minimal -= 1 return g def OutputFreq(self, termFreqList, minimal=None): _freq = [] for term in termFreqList: _freq.append(termFreqList[term]) freq = numpy.array(_freq) if minimal is None: minimal = freq.mean() + freq.std() for term in sorted(termFreqList, key=termFreqList.get, reverse=True): if termFreqList[term] > minimal: print('%s:%d' % (term, termFreqList[term]), end=' ') print() def OutputDistanceFreq(self, termFreqList, minimal=None): _freq = [] for term in termFreqList: _freq.append(termFreqList[term]) freq = numpy.array(_freq) if minimal is None: minimal = math.ceil(freq.mean() + freq.std()) for term in sorted(termFreqList, key=termFreqList.get, reverse=True): if termFreqList[term] > minimal: print('%s:%d' % (term, termFreqList[term] - minimal)) print() def Output(self, content): for word in content: print(word, end=' ') print() if __name__ == '__main__': fileName = 'data/1.txt' if len(sys.argv) > 1: fileName = sys.argv[1] worker = TextFileProcessing() content = None termFreqList = None taskControl = [False, False, True] if taskControl[0]: content, termFreqList = worker.LoadFile(fileName) else: content, termFreqList = worker.LoadFile2Distance(fileName) if taskControl[1]: worker.Output(content) if taskControl[2]: if taskControl[0]: worker.OutputFreq(termFreqList) else: worker.OutputDistanceFreq(termFreqList)