#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import codecs
import string
import re
import sqlite3

class NewsParser():
    DataPath = ""
    NewsPaper = ""
    NewsThread = ""
    NewsDictDB = ""
    DbConnection = None
    
    def __init__(self, DataPath, NewsPaper, NewsThread, NewsDictDB):
        self.DataPath = DataPath
        self.NewsPaper = NewsPaper
        self.NewsThread = NewsThread
        self.NewsDictDB = NewsDictDB
        
    def SeparateChineseDelimiters(self, ChineseWords):
        ChineseLines = []
        WordList1 = ChineseWords.split(u"。")
        for Words1 in WordList1:
            WordList2 = Words1.split(u"？")
            for Words2 in WordList2:
                WordList3 = Words2.split(u"！")
                for Words3 in WordList3:
                   WordList4 = Words3.split(u"，")
                   for Words4 in WordList4:
                       WordList5 = Words4.split(u"、")
                       for Words5 in WordList5:
                           WordList6 = Words5.split(u"：")
                           for Words6 in WordList6:
                               ChineseLines.append(Words6)
                  
        return(ChineseLines)
     
    def SeparateEnglishDelimiters(self, EnglishWords):
        EnglishLines = []
        WordList = re.split("[.|,|?|!]", EnglishWords)
        for Words in WordList:
           EnglishLines.append(Words)
                  
        return(EnglishLines)
        
    def CheckDictionary(self, NewsWords):
        DbSQL = "SELECT Term, LENGTH(TERM) AS LTERM FROM chinese WHERE Term LIKE '%s%%' ORDER BY LTERM DESC" % (NewsWords)
        DbRows = self.DbConnection.execute(DbSQL)
    
        DbFound = False
        TermList = []
        for DbRow in DbRows:
            TermList.append(DbRow[0])
            DbFound = True
            
        DbRows = None    
        return(DbFound, TermList) 
        
    def ProcessNewsLine(self, NewsLine):
        NewsWords = NewsLine.split()
        NewsWordList = []
        i = 0
        while i < len(NewsWords):
            (DbFound, TermList)  =  self.CheckDictionary(NewsWords[i])
            if DbFound == False:
                 NewsWordList.append(NewsWords[i])
                 i += 1
                 continue
    
            Found = False
            for Term in TermList:
                j = 0
                TermLength = 0
                Words = []
                while (TermLength < len(Term)) and (i+j < len(NewsWords)):
                    TermLength += len(NewsWords[i + j])
                    if (TermLength > len(Term)) :
                        break
                    
                    Words.append(NewsWords[i + j])                
                    j += 1
    
                if string.join(Words, '') == Term:
                    NewsWordList.append(Term)
                    i += j
                    Found = True
                    break
            
            if not Found:
                NewsWordList.append(NewsWords[i])
                i += 1
                
        return(NewsWordList) 
    
    def doTask(self):
        self.DbConnection = sqlite3.connect(NewsDictionay)
        NewsFileHandle = codecs.open(NewsFile, "r", encoding="utf-8")
        
        for NewsParagraph in NewsFileHandle:
            NewsLineList =  self.SeparateChineseDelimiters(NewsParagraph)
            for NewsLine in NewsLineList:
                TermList = self.ProcessNewsLine(NewsLine)
                for Term in TermList:
                     print codecs.encode(Term, "utf-8"), '\t',
                
                print
    
        NewsFileHandle.close()
        self.DbConnection.close()    

if __name__ == "__main__":
    DataPath = "/home/richclee/Desktop/News"
    NewsPaper = "ChinaTimes"
    NewsThread = "0320"
    NewsDictDB = "Dictionaries/NewsDict.db"
    
    if len(sys.argv) > 1:
        NewsThread = sys.argv[1]
        
    if len(sys.argv) > 2:
        NewsPaper = sys.argv[2]
        
    NewsFile = "%s/%s/%s.txt" % (DataPath, NewsPaper, NewsThread)
    NewsDictionay = "%s/%s" % (DataPath, NewsDictDB)
    
    newsParser = NewsParser(DataPath, NewsPaper, NewsThread, NewsDictDB)    
    newsParser.doTask()