def basic_tokenize(text):
    """基礎(chǔ)空格分詞"""
    return text.split()

# 測(cè)試
text = "Python is an interpreted programming language"
tokens = basic_tokenize(text)
# ['Python', 'is', 'an', 'interpreted', 'programming', 'language']

import re

def regex_tokenize(text):
    """正則表達(dá)式分詞"""
    # 匹配單詞和基本標(biāo)點(diǎn)
    pattern = r'\w+|[^\w\s]'
    return re.findall(pattern, text)

# 測(cè)試
text = "Hello, world! How are you?"
tokens = regex_tokenize(text)
# ['Hello', ',', 'world', '!', 'How', 'are', 'you', '?']

def advanced_regex_tokenize(text):
    """處理復(fù)雜文本的分詞"""
    # 匹配：?jiǎn)卧~、連字符詞、縮寫、貨幣、表情符號(hào)
    pattern = r"""
        \d+\.\d+          | # 浮點(diǎn)數(shù)
        \d+,\d+           | # 千位分隔數(shù)字
        \d+               | # 整數(shù)
        \w+(?:-\w+)+      | # 連字符詞
        [A-Z]+\.[A-Z]+\.?| # 縮寫 (U.S.A.)
        \$\d+             | # 貨幣
        [\U0001F600-\U0001F64F] | # 表情符號(hào)
        \w+               | # 單詞
        [^\w\s]             # 標(biāo)點(diǎn)符號(hào)
    """
    return re.findall(pattern, text, re.VERBOSE | re.UNICODE)

# 測(cè)試
text = "I paid $99.99 for this item in the U.S.A. ??"
tokens = advanced_regex_tokenize(text)
# ['I', 'paid', '$99.99', 'for', 'this', 'item', 'in', 'the', 'U.S.A.', '??']

import nltk
nltk.download('punkt')

from nltk.tokenize import word_tokenize, sent_tokenize

# 句子分詞
sentences = sent_tokenize("First sentence. Second sentence!")
# ['First sentence.', 'Second sentence!']

# 單詞分詞
tokens = word_tokenize("Python's nltk module is powerful!")
# ['Python', "'s", 'nltk', 'module', 'is', 'powerful', '!']

from nltk.tokenize import TweetTokenizer, MWETokenizer

# 推特分詞器（處理表情符號(hào)和@提及）
tweet_tokenizer = TweetTokenizer()
tokens = tweet_tokenizer.tokenize("OMG! This is so cool ?? #NLP @nlp_news")
# ['OMG', '!', 'This', 'is', 'so', 'cool', '??', '#NLP', '@nlp_news']

# 多詞表達(dá)分詞器
mwe_tokenizer = MWETokenizer([('New', 'York'), ('machine', 'learning')])
tokens = mwe_tokenizer.tokenize("I live in New York and study machine learning".split())
# ['I', 'live', 'in', 'New_York', 'and', 'study', 'machine_learning']

import spacy

# 加載模型
nlp = spacy.load("en_core_web_sm")

# 分詞處理
doc = nlp("Apple's stock price rose $5.45 to $126.33 in pre-market trading.")
tokens = [token.text for token in doc]
# ['Apple', "'s", 'stock', 'price', 'rose', '$', '5.45', 'to', '$', '126.33', 'in', 'pre', '-', 'market', 'trading', '.']

def analyze_tokens(doc):
    """獲取分詞詳細(xì)信息"""
    token_data = []
    for token in doc:
        token_data.append({
            "text": token.text,
            "lemma": token.lemma_,
            "pos": token.pos_,
            "tag": token.tag_,
            "dep": token.dep_,
            "is_stop": token.is_stop,
            "is_alpha": token.is_alpha,
            "is_digit": token.is_digit
        })
    return token_data

# 測(cè)試
doc = nlp("The quick brown fox jumps over the lazy dog.")
token_info = analyze_tokens(doc)

from spacy.tokenizer import Tokenizer
from spacy.util import compile_prefix_regex, compile_suffix_regex

def create_custom_tokenizer(nlp):
    """創(chuàng)建自定義分詞器"""
    # 自定義前綴規(guī)則（處理$等特殊前綴）
    prefixes = nlp.Defaults.prefixes + [r'\$']
    prefix_regex = compile_prefix_regex(prefixes)
    
    # 自定義后綴規(guī)則
    suffixes = nlp.Defaults.suffixes + [r'\%']
    suffix_regex = compile_suffix_regex(suffixes)
    
    # 自定義分詞規(guī)則
    rules = nlp.Defaults.tokenizer_exceptions
    rules.update({
        "dont": [{"ORTH": "dont"}],  # 不分詞
        "can't": [{"ORTH": "can"}, {"ORTH": "'t"}]  # 特殊分詞
    })
    
    return Tokenizer(
        nlp.vocab,
        rules=rules,
        prefix_search=prefix_regex.search,
        suffix_search=suffix_regex.search,
        infix_finditer=nlp.Defaults.infix_finditer,
        token_match=nlp.Defaults.token_match
    )

# 使用自定義分詞器
nlp = spacy.load("en_core_web_sm")
nlp.tokenizer = create_custom_tokenizer(nlp)

doc = nlp("I don't like $100% increases.")
tokens = [token.text for token in doc]
# ['I', 'dont', 'like', '$100%', 'increases', '.']

import jieba
import jieba.posseg as pseg

# 基礎(chǔ)分詞
text = "自然語言處理是人工智能的重要方向"
words = jieba.cut(text)
print("/".join(words))  # "自然語言/處理/是/人工智能/的/重要/方向"

# 全模式
words_full = jieba.cut(text, cut_all=True)
# "自然/自然語言/語言/處理/是/人工/人工智能/智能/重要/方向"

# 搜索引擎模式
words_search = jieba.cut_for_search(text)
# "自然/語言/自然語言/處理/是/人工/智能/人工智能/重要/方向"

# 添加自定義詞典
jieba.add_word("自然語言處理")
jieba.add_word("人工智能")

# 加載自定義詞典文件
jieba.load_userdict("custom_dict.txt")

# 詞性標(biāo)注
words = pseg.cut(text)
for word, flag in words:
    print(f"{word} ({flag})")
    
# 自然語言處理 (n)
# 是 (v)
# 人工智能 (n)
# 的 (uj)
# 重要 (a)
# 方向 (n)

def medical_tokenizer(text):
    """醫(yī)學(xué)文本分詞器"""
    # 加載基礎(chǔ)模型
    nlp = spacy.load("en_core_sci_sm")
    
    # 添加醫(yī)學(xué)詞典
    with open("medical_terms.txt") as f:
        for term in f:
            nlp.tokenizer.add_special_case(term, [{"ORTH": term}])
    
    # 處理醫(yī)學(xué)縮寫
    abbreviations = {
        "CVD": "cardiovascular disease",
        "MI": "myocardial infarction"
    }
    
    # 處理醫(yī)學(xué)復(fù)合詞
    compound_rules = [
        ("blood", "pressure"),
        ("heart", "rate"),
        ("red", "blood", "cell")
    ]
    
    for terms in compound_rules:
        nlp.tokenizer.add_special_case(
            " ".join(terms),
            [{"ORTH": "_".join(terms)}]
        )
    
    return nlp(text)

# 使用
text = "Patient with CVD and high blood pressure. History of MI."
doc = medical_tokenizer(text)
tokens = [token.text for token in doc]
# ['Patient', 'with', 'CVD', 'and', 'high_blood_pressure', '.', 'History', 'of', 'MI', '.']

def legal_tokenizer(text):
    """法律文本分詞器"""
    # 基礎(chǔ)分詞
    nlp = spacy.load("en_core_web_sm")
    
    # 添加法律術(shù)語
    legal_terms = [
        "force majeure",
        "prima facie",
        "pro bono",
        "voir dire"
    ]
    
    for term in legal_terms:
        nlp.tokenizer.add_special_case(
            term,
            [{"ORTH": term.replace(" ", "_")}]
        )
    
    # 處理法律引用
    pattern = r"(\d+)\s+(U\.S\.C\.|U\.S\.)\s+§\s+(\d+)"
    text = re.sub(pattern, r"\1_\2_§_\3", text)
    
    return nlp(text)

# 使用
text = "As per 42 U.S.C. § 1983, the plaintiff..."
doc = legal_tokenizer(text)
tokens = [token.text for token in doc]
# ['As', 'per', '42_U.S.C._§_1983', ',', 'the', 'plaintiff', '...']

class StreamTokenizer:
    """流式分詞處理器"""
    def __init__(self, tokenizer_func, buffer_size=4096):
        self.tokenizer = tokenizer_func
        self.buffer = ""
        self.buffer_size = buffer_size
    
    def process(self, text_chunk):
        """處理文本塊"""
        self.buffer += text_chunk
        tokens = []
        
        # 處理完整句子
        while '.' in self.buffer or '!' in self.buffer or '?' in self.buffer:
            # 查找最近的句子結(jié)束符
            end_pos = min(
                self.buffer.find('.'),
                self.buffer.find('!'),
                self.buffer.find('?')
            )
            if end_pos == -1:
                break
                
            # 提取句子
            sentence = self.buffer[:end_pos+1]
            self.buffer = self.buffer[end_pos+1:]
            
            # 分詞
            tokens.extend(self.tokenizer(sentence))
        
        return tokens
    
    def finalize(self):
        """處理剩余文本"""
        if self.buffer:
            tokens = self.tokenizer(self.buffer)
            self.buffer = ""
            return tokens
        return []

# 使用示例
tokenizer = StreamTokenizer(word_tokenize)
with open("large_text.txt") as f:
    while chunk := f.read(1024):
        tokens = tokenizer.process(chunk)
        process_tokens(tokens)  # 處理分詞結(jié)果
    
# 處理剩余內(nèi)容
final_tokens = tokenizer.finalize()
process_tokens(final_tokens)

from flask import Flask, request, jsonify
import threading
import spacy

app = Flask(__name__)

# 預(yù)加載模型
nlp = spacy.load("en_core_web_sm")

# 請(qǐng)求隊(duì)列
request_queue = []
result_dict = {}
lock = threading.Lock()

def tokenizer_worker():
    """分詞工作線程"""
    while True:
        if request_queue:
            with lock:
                req_id, text = request_queue.pop(0)
            
            # 處理分詞
            doc = nlp(text)
            tokens = [token.text for token in doc]
            
            with lock:
                result_dict[req_id] = tokens

# 啟動(dòng)工作線程
threading.Thread(target=tokenizer_worker, daemon=True).start()

@app.route('/tokenize', methods=['POST'])
def tokenize_endpoint():
    """分詞API端點(diǎn)"""
    data = request.json
    text = data.get('text', '')
    req_id = id(text)
    
    with lock:
        request_queue.append((req_id, text))
    
    # 等待結(jié)果
    while req_id not in result_dict:
        time.sleep(0.01)
    
    with lock:
        tokens = result_dict.pop(req_id)
    
    return jsonify({"tokens": tokens})

# 啟動(dòng)服務(wù)
if __name__ == '__main__':
    app.run(threaded=True, port=5000)

from collections import Counter
from string import punctuation

def extract_keywords(text, top_n=10):
    """提取關(guān)鍵詞"""
    # 分詞
    doc = nlp(text)
    
    # 過濾停用詞和標(biāo)點(diǎn)
    words = [
        token.text.lower() 
        for token in doc 
        if not token.is_stop and not token.is_punct and token.is_alpha
    ]
    
    # 計(jì)算詞頻
    word_freq = Counter(words)
    return word_freq.most_common(top_n)

# 測(cè)試
text = "Python is an interpreted high-level programming language. Python is widely used in data science."
keywords = extract_keywords(text)
# [('python', 2), ('interpreted', 1), ('high', 1), ('level', 1), ('programming', 1), ('language', 1), ('widely', 1), ('used', 1), ('data', 1), ('science', 1)]

def preprocess_for_sentiment(text):
    """情感分析預(yù)處理"""
    # 分詞
    doc = nlp(text)
    
    # 預(yù)處理步驟
    tokens = []
    for token in doc:
        # 小寫化
        token_text = token.text.lower()
        
        # 移除停用詞
        if token.is_stop:
            continue
        
        # 詞形還原
        lemma = token.lemma_
        
        # 移除標(biāo)點(diǎn)
        if lemma in punctuation:
            continue
        
        tokens.append(lemma)
    
    return tokens

# 測(cè)試
text = "I really love this product! It's amazing."
processed = preprocess_for_sentiment(text)
# ['really', 'love', 'product', 'amazing']

import timeit

text = "Natural language processing is a subfield of linguistics, computer science, and artificial intelligence."

# 測(cè)試函數(shù)
def test_regex():
    return regex_tokenize(text)

def test_nltk():
    return word_tokenize(text)

def test_spacy():
    doc = nlp(text)
    return [token.text for token in doc]

def test_jieba():
    return list(jieba.cut(text))

# 性能測(cè)試
methods = {
    "Regex": test_regex,
    "NLTK": test_nltk,
    "spaCy": test_spacy,
    "jieba": test_jieba
}

results = {}
for name, func in methods.items():
    time = timeit.timeit(func, number=1000)
    results[name] = time

print("1000次分詞操作耗時(shí):")
for name, time in sorted(results.items(), key=lambda x: x[1]):
    print(f"{name}: {time:.4f}秒")

def preprocess(text):
    # 小寫化
    text = text.lower()
    # 移除特殊字符
    text = re.sub(r'[^\w\s]', '', text)
    # 分詞
    return word_tokenize(text)

from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))

def remove_stopwords(tokens):
    return [t for t in tokens if t not in stop_words]

from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()

def lemmatize(tokens):
    return [lemmatizer.lemmatize(t) for t in tokens]

# 預(yù)加載模型
nlp = spacy.load("en_core_web_sm")

# 批量處理
texts = ["text1", "text2", "text3"]
docs = list(nlp.pipe(texts))

# 添加領(lǐng)域術(shù)語
nlp.tokenizer.add_special_case("machine_learning", [{"ORTH": "machine_learning"}])

try:
    tokens = tokenize(text)
except TokenizationError as e:
    logger.error(f"Tokenization failed: {str(e)}")
    tokens = fallback_tokenize(text)

class TestTokenization(unittest.TestCase):
    def test_basic_tokenization(self):
        tokens = tokenize("Hello, world!")
        self.assertEqual(tokens, ["Hello", ",", "world", "!"])
    
    def test_domain_term(self):
        tokens = tokenize("machine learning")
        self.assertEqual(tokens, ["machine_learning"])