英文稿分析-2

#!/usr/bin/env python3# -*- coding: utf-8 -*-import jsonimport loggingimport sysimport codecsimport stringimport nltkclass TextFilter: __numList__ = None __puncList__ = None __stopWords__ = None __trivialWords__ = [] __params__ = { 'minimalChars': 4 } def __init__(self, params): logger = logging.getLogger('Text Ming') logging.basicConfig(format='%(asctime)s %(message)s') logger.setLevel(logging.DEBUG) # Ignore Less Meaning Word List if 'ignoredWords' in params: self.__trivialWords__.clear() with codecs.open(params['ignoredWords'], 'r', 'utf-8') as textFifle: logging.debug('Ignored Work List: %s' % params['ignoredWords']) for line in textFifle: _line = line.strip().lower() for term in _line.split(): self.__trivialWords__.append(term) self.__numList__ = string.digits self.__puncList__ = string.punctuation self.__stopWords__ = set(nltk.corpus.stopwords.words('english')) # Minimal Characters Accepted if 'minimalChars' in params: self.__params__['minimalChars'] = params['minimalChars'] logging.debug('Minimal Chars Accepted: %d' % params['minimalChars']) # Filter Trivial Characters & Words def __filterOut__(self, word): _word = word for ch in self.__puncList__: _word = _word.replace(ch, '') for ch in self.__numList__: _word = _word.replace(ch, '') if _word in self.__stopWords__: _word = '' if _word in self.__trivialWords__: _word = '' return _word # Check Word in English Dictionary def __isWord__(self, word): return len(nltk.corpus.wordnet.synsets(word)) > 0 # Read Text Document Generating Term Frequency Table def LoadFile(self, fileName): termInfoList = {} stemmer = nltk.stem.snowball.SnowballStemmer('english') with codecs.open(fileName, 'r', 'utf-8') as textFifle: for line in textFifle: _line = line.strip().lower() _wordList = nltk.word_tokenize(_line) for word in _wordList: _word = self.__filterOut__(word) if self.__isWord__(_word): if len(_word) >= self.__params__['minimalChars']: # Unified Word Form _wordStemmed = stemmer.stem(_word) if _word not in termInfoList: termInfoList[_wordStemmed] = { 'count': 1, 'wordList': [ { 'word': _word, 'count': 1 } ] } else: _dictTerm = termInfoList[_wordStemmed] _dictTerm['count'] += 1 _found = False for _wordInfo in _dictTerm['wordList']: if _wordInfo['word'] == _word: _wordInfo['count'] += 1 _found = True break if not _found: _dictTerm['wordList'].append( { 'word': _word, 'count': 1 }) return termInfoList # Display Content Term Info def OutputContentInfo(self, termInfoList): for wordStemmed in termInfoList: print('%s:%d' % (wordStemmed, termInfoList[wordStemmed]['count'])) for _wordInfo in termInfoList[wordStemmed]['wordList']: print('\t%s:%d' % (_wordInfo['word'], _wordInfo['count'])) # Display Content Term Frequency def OutputContentFreq(self, termInfoList, csvFile=None): _list = sorted(termInfoList.items(), key=lambda x: x[1]['count'], reverse=True) textFile = None if csvFile is not None: textFile = codecs.open(csvFile, 'w', 'utf-8') textFile.write('Term,Word,Freq\n') for wordStemmedInfo in _list: # ('european', {'count': 7, 'wordList': [{'word': 'european', 'count': 7}]}) _wordStemmed, _termInfo = wordStemmedInfo _maxWordCount = 0 _word = _wordStemmed for _wordInfo in _termInfo['wordList']: if _wordInfo['count'] > _maxWordCount: _word = _wordInfo['word'] _maxWordCount = _wordInfo['count'] if csvFile is not None: textFile.write('%s,%s,%d\n' % (_wordStemmed, _word, _termInfo['count'])) else: print('%s:%s:%d' % (_wordStemmed, _word, _termInfo['count'])) if csvFile is not None: textFile.close() # Mainif __name__ == '__main__': configFile = 'config.json' if len(sys.argv) > 1: configFile = sys.argv[1] config = {} with codecs.open(configFile, 'r', 'utf-8') as jsonFile: config = json.load(jsonFile) worker = TextFilter(config) taskControl = config['task'] # Read Document textFile = 'data/%s.txt' % (config['sourceDoc']) termInfoList = worker.LoadFile(textFile) if taskControl[0]: # Display Content Term Info worker.OutputContentInfo(termInfoList) if taskControl[1]: # Display Content Term Frequency csvFile = 'result/%s.csv' % (config['sourceDoc']) worker.OutputContentFreq(termInfoList, csvFile)