去除txt重複文本的python code
Python code for removing duplicate txt
def remove_duplicated_content(input_file, output_file):
# 讀取文件內容
with open(input_file, 'r', encoding='utf-8') as f:
content = f.read()
# 按分頁標記分割文本
sections = content.split('--- 分頁 ---')
# 使用集合來去除重複段落,同時保持順序
unique_sections = []
seen_content = set()
for section in sections:
# 處理每個部分
paragraphs = section.split('\n\n')
unique_paragraphs = []
for paragraph in paragraphs:
# 忽略空段落
if not paragraph.strip():
continue
# 如果段落不在已見集合中,則添加到結果中
if paragraph not in seen_content:
seen_content.add(paragraph)
unique_paragraphs.append(paragraph)
# 只有當有內容時才添加該部分
if unique_paragraphs:
unique_section = '\n\n'.join(unique_paragraphs)
unique_sections.append(unique_section)
# 將去重後的部分重新組合成文本
unique_content = '\n\n--- 分頁 ---\n\n'.join(unique_sections)
# 寫入輸出文件
with open(output_file, 'w', encoding='utf-8') as f:
f.write(unique_content)
print(f"處理完成!原文件有 {len(sections)} 個部分,處理後有 {len(unique_sections)} 個部分。")
# 使用範例
input_file = "原子習慣.txt"
output_file = "原子習慣_去重.txt"
remove_duplicated_content(input_file, output_file)