NLP.Word Tokenization

 

from nltk.tokenize import word_tokenize  
print(word_tokenize("Local farmers are protesting strengthened regulations on living shelters for foreign laborers introduced in the wake of a Cambodian worker’s death, claiming the measures are ill-devised and demanding more support from the government."))

'''
['Local', 'farmers', 'are', 'protesting', 'strengthened', 'regulations', 'on', 'living', 'shelters', 'for', 'foreign', 'laborers', 'introduced', 'in', 'the', 'wake', 'of', 'a', 'Cambodian', 'worker', '’', 's', 'death', ',', 'claiming', 'the', 'measures', 'are', 'ill-devised', 'and', 'demanding', 'more', 'support', 'from', 'the', 'government', '.']
'''

 

from nltk.tokenize import WordPunctTokenizer  
print(WordPunctTokenizer().tokenize("Local farmers are protesting strengthened regulations on living shelters for foreign laborers introduced in the wake of a Cambodian worker’s death, claiming the measures are ill-devised and demanding more support from the government."))

'''
['Local', 'farmers', 'are', 'protesting', 'strengthened', 'regulations', 'on', 'living', 'shelters', 'for', 'foreign', 'laborers', 'introduced', 'in', 'the', 'wake', 'of', 'a', 'Cambodian', 'worker', '’', 's', 'death', ',', 'claiming', 'the', 'measures', 'are', 'ill', '-', 'devised', 'and', 'demanding', 'more', 'support', 'from', 'the', 'government', '.']
'''

 

from tensorflow.keras.preprocessing.text import text_to_word_sequence
print(text_to_word_sequence("Local farmers are protesting strengthened regulations on living shelters for foreign laborers introduced in the wake of a Cambodian worker’s death, claiming the measures are ill-devised and demanding more support from the government."))

'''
['local', 'farmers', 'are', 'protesting', 'strengthened', 'regulations', 'on', 'living', 'shelters', 'for', 'foreign', 'laborers', 'introduced', 'in', 'the', 'wake', 'of', 'a', 'cambodian', 'worker’s', 'death', 'claiming', 'the', 'measures', 'are', 'ill', 'devised', 'and', 'demanding', 'more', 'support', 'from', 'the', 'government']
'''

 

from nltk.tokenize import TreebankWordTokenizer
tokenizer=TreebankWordTokenizer()
text="We also have passed all inspections when obtaining our employment permits, and fire safety devices are installed inside each cargo container. While the death of the foreign worker is to be mourned, many farmers will agree with me that the situation is only a very extreme and minor view of the farming sector.By Ko Jun-tae (ko.juntae@heraldcorp.com)"
print(tokenizer.tokenize(text))

'''
['We', 'also', 'have', 'passed', 'all', 'inspections', 'when', 'obtaining', 'our', 'employment', 'permits', ',', 'and', 'fire', 'safety', 'devices', 'are', 'installed', 'inside', 'each', 'cargo', 'container.', 'While', 'the', 'death', 'of', 'the', 'foreign', 'worker', 'is', 'to', 'be', 'mourned', ',', 'many', 'farmers', 'will', 'agree', 'with', 'me', 'that', 'the', 'situation', 'is', 'only', 'a', 'very', 'extreme', 'and', 'minor', 'view', 'of', 'the', 'farming', 'sector.By', 'Ko', 'Jun-tae', '(', 'ko.juntae', '@', 'heraldcorp.com', ')']
'''