[Data] Frequency Analysis for Covid19 Keywords from Twitter, Naver & Creating WordCloud

!pip install konlpy

import numpy as np 
import pandas as pd
import matplotlib.pyplot as plt

from konlpy.tag import Twitter
from collections import Counter
from wordcloud import WordCloud
from tqdm import tqdm

naver_emo = pd.read_excel("/content/Naver/NAVER_코로나, 감정.xlsx")
naver_mood = pd.read_excel("/content/Naver/NAVER_코로나, 기분.xlsx")
naver_daily = pd.read_excel("/content/Naver/NAVER_코로나, 일상.xlsx")

twitter_emo = pd.read_excel("/content/Twitter/Tweets_코로나, 감정.xlsx")
twitter_mood = pd.read_excel("/content/Twitter/Tweets_코로나, 기분.xlsx")
twitter_daily = pd.read_excel("/content/Twitter/Tweets_코로나, 일상.xlsx")

naver_emo.rename(columns={'Tweets' : 'Contents'}, inplace=True)
naver_mood.rename(columns={'Tweets' : 'Contents'}, inplace=True)
naver_daily.rename(columns={'Tweets' : 'Contents'}, inplace=True)

naver_daily

remove = ["없었어요", "었어요", "한번", "적", "것", "한", "하게", "한게", "해요", "합니다",\
          "하여", "해서", "했습니다", "하나요", "할까요", "해가지고", "하니까", "하다고", "했는데"\
          "하면", "할까요", "하고", "했어요", "져서요", "집니다", "납니다", "하지만", "지만",\
          "할", "한걸까요", "까요", "입니다", "어떻게", "없어서", "그래서", "그런데", "그런", "근데", "같이",\
          "가끔", "너무", "때문", "때문에", "자꾸", "있을", "있는", "요즘", "약간", "영어",\
          "이런", "왜", "이럴까요", "저희", "우리", "많이", "계속", "다시", "이틀", "지금", "코로나", "백신",\
          "그냥", "부터", "승무원", '해도', "언젠가", "있으면", "시어머니", "아버님", "통장", "있고", "문장",\
          "현재", "치매", "해도", "있으면", "있고", "감정", "있었음", "있어요",\
          "그래도", "진자", "오늘", "같은", "기분", "진짜", "정말", "이번", "없는", "이렇게", "없이", "일상", "이제"]

def get_morphs(df_naver, df_twitter):
    twt = Twitter()

    global remove
    morphs = []
    morphs_twt = []

    for sentence in tqdm(df_naver['Contents']):
        morph = twt.pos(sentence)
        morphs.append(morph)

    for sentence in tqdm(df_twitter['Tweets']):
        morph = twt.pos(sentence)
        morphs_twt.append(morph)

    return morphs, morphs_twt 

def get_freqs_all(morphs, morphs_twt, keyword):
    words = []
    for sentence in tqdm(morphs):
        for elem in sentence:
          if elem[1] in ["Noun", "Adjective", "Adverb"]:
              tmp = elem[0]
              for r in remove:
                  tmp = tmp.replace(r, "")
              if tmp:
                words.append(tmp)

    words_twt = []
    for sentence in tqdm(morphs_twt):
        for elem in sentence:
          if elem[1] in ["Noun", "Adjective", "Adverb"]:
              tmp = elem[0]
              for r in remove:
                  tmp = tmp.replace(r, "")
              if tmp:
                words_twt.append(tmp)

    words = [word for word in words if len(word) > 1]
    words_twt = [word for word in words_twt if len(word) > 1]

    words_total = words + words_twt

    counter_total = Counter(words_total)
    freqs_total = counter_total.most_common(500)
    print(freqs_total[:30])

    freqs_total_df = pd.DataFrame(freqs_total, columns=['키워드', '빈도수'])
    freqs_total_df.to_excel(keyword + "_total_freq.xlsx")

    return freqs_total

!apt -qq -y install fonts-nanum > /dev/null

import matplotlib.font_manager as fm
fontpath="/usr/share/fonts/truetype/nanum/NanumBarunGothic.ttf"
font = fm.FontProperties(fname=fontpath)

from wordcloud import ImageColorGenerator
from PIL import Image

mask = np.array(Image.open('/content/cloud2.jpg'))
colors = ImageColorGenerator(mask)

def get_wordcloud(freqs, add_stopwords):
    stopwords = ["기초", "같고", "자주", "인해", "아이"]
    stopwords += add_stopwords
    
    wordcloud = WordCloud(font_path=fontpath, 
                          background_color="black",
                          relative_scaling=0.5,
                          colormap='tab20c',
                          mask = mask,
                          stopwords=stopwords,
                          contour_color='black',
                          )
    cloud = wordcloud.generate_from_frequencies(dict(freqs))
    plt.figure(figsize=(25, 8))
    plt.axis("off")
    plt.imshow(cloud)
    plt.show()

morphs_emo, morph_emo_twt = get_morphs(naver_emo, twitter_emo)

morphs_emo
morph_emo_twt

emo_freqs = get_freqs_all(morphs_emo, morph_emo_twt, "감정")

emo_freqs[:30]

get_wordcloud(emo_freqs, [])

morphs, morphs_twt = get_morphs(naver_mood, twitter_mood)

morphs
morphs_twt

mood_freqs = get_freqs_all(morphs, morphs_twt, "기분")

get_wordcloud(mood_freqs, ["그래도", "진자", "오늘", "같은", "기분", "진짜", "정말", "이번", "없는", "이렇게", "없이"])

morphs_daily, morphs_daily_twt = get_morphs(naver_daily, twitter_daily)

morphs_daily
morphs_daily_twt

daily_freqs = get_freqs_all(morphs_daily, morphs_daily_twt, "일상")

daily_freqs[:30]

get_wordcloud(daily_freqs, ["일상", "이제"])