Data Cleaning_2

Noise_word_removal.py

freq_words=df.Treated_Tweet.str.split(expand=True).stack().value_counts()[:10]
freq_words=list(freq_words.index)
rare_words=df.Treated_Tweet.str.split(expand=True).stack().value_counts()
rare_words=list(rare_words.loc[lambda x: x==1].index)


#Remove Frequent and Rare words
def remove_noise_words(text):
    edited_text=text.split()
    edited_text=[word for word in edited_text if word not in freq_words]
    edited_text=[word for word in edited_text if word not in rare_words]
    edited_text=" ".join(edited_text)
    return edited_text