텍스트 마이닝 프로젝트 : 중고거래 분석 (중고나라)

from selenium import webdriver
from selenium_crawler import main as crawler
import json

"""
Default selenium webdriver setting

variables:
headless_flag   -- selenium chrome browser headless or not
options         -- selenium chrome browser options
"""

headless_flag = True

if headless_flag:
    options = webdriver.ChromeOptions()
    options.add_argument('headless')
    options.add_argument('window-size=1920x1080')
    options.add_argument("disable-gpu")
    driver = webdriver.Chrome('chromedriver', options=options)
else:
    driver = webdriver.Chrome('chromedriver')

# Wait until driver load resources
driver.implicitly_wait(3)

"""
Main crawling task

file:
selenium_crawler.py

module:
main(location, link, filename, driver)    -- main crawler

main arguments:
location    -- korean location name
link        -- browser menu id
filename    -- path result stored
driver      -- selenium driver
"""

with open('source/location_data.json') as loca_data:
    ld = json.load(loca_data)

for d in ld:
    crawler(d['loca'], d['link'], d['filename'], driver)

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import codecs

def main(location, link, filename, driver):
    driver.get('http://cafe.naver.com/joonggonara')
    wait = WebDriverWait(driver, 30)
    wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "a.gm-tcol-c")))

    location_group = driver.find_element_by_css_selector("a[title*='" + location + "']")
    driver.execute_script("arguments[0].scrollIntoView();", location_group)
    location_group.click();

    elm = driver.find_element_by_id(link)
    driver.execute_script("arguments[0].scrollIntoView();", elm)
    elm.click()

    iframe_element = driver.find_element_by_css_selector("iframe#cafe_main")
    driver.switch_to.frame(iframe_element)

    first = 0
    end = 11
    page_10th = 99

    f = codecs.open(filename+"(2019).txt", 'w', encoding='utf-8')
    fo = codecs.open(filename + "(~2019).txt", 'w', encoding='utf-8')

    file_switch_flag = False

    notice_hidden = driver.find_element_by_css_selector("label[for*='notice_hidden']")
    driver.execute_script("arguments[0].scrollIntoView();", notice_hidden)
    notice_hidden.click()

    for ten in range(page_10th):
        for i in range(first, end):
            page_button = driver.find_element_by_class_name("prev-next")
            driver.execute_script("arguments[0].scrollIntoView();", page_button)
            page_buttons = driver.find_elements_by_css_selector("div.prev-next a")

            if (ten is not 0) and (len(page_buttons) <= 11) and (i is len(page_buttons)):
                debug_last_date = driver.find_elements_by_css_selector("td.td_date")
                if file_switch_flag:
                    fo.write("last date : " + debug_last_date[0].text)
                else:
                    f.write("last date : " + debug_last_date[0].text)

                print("last date : " + debug_last_date[0].text)
                print("DEBUG TERMINATION")
                return 0;

            page_buttons[i].click()

            if i is end-1:
                first = 1
                end = 12
            else:
                print("------------" + filename + " " + str(10 * ten + i) + "page-----------")
                articles = driver.find_elements_by_css_selector("td.td_article")
                date = driver.find_elements_by_css_selector("td.td_date")[0].text

                if date[:4] == "2018":
                    file_switch_flag = True

                for article in articles:
                    title_text = article.find_element_by_css_selector('a.article').text
                    if file_switch_flag:
                        fo.write(title_text+"\n")
                    else:
                        f.write(title_text+"\n")

new_nouns = []

with open('preprocess/dictionary.txt', encoding='utf8') as fd:
    for line in fd:
        new_nouns.append(line.strip('\n'))

twitter.add_dictionary(new_nouns, 'Noun')

from ckonlpy.tag import Postprocessor
from ckonlpy.tag import Twitter
from ckonlpy.utils import load_wordset
from ckonlpy.utils import load_replace_wordpair
from ckonlpy.utils import load_ngram

passwords = load_wordset('postprocess/passwords.txt')
stopwords = load_wordset('postprocess/stopwords.txt')
replace = load_replace_wordpair('postprocess/replace.txt')
ngrams = load_ngram('postprocess/ngrams.txt')

twitter = Twitter()
new_nouns = []

with open('preprocess/dictionary.txt', encoding='utf8') as fd:
    for line in fd:
        new_nouns.append(line.strip('\n'))

twitter.add_dictionary(new_nouns, 'Noun')

passtags = {'Noun'}

postprocessor = Postprocessor(
    base_tagger = twitter,
    stopwords = stopwords,
    passtags = passtags,
    replace = replace,
    ngrams = ngrams
)

filename = '성동구(~2019)'

with open('outputs/서울/' + filename + '.txt', 'r') as f:
    lines = f.read()
    pre_nouns = postprocessor.pos(lines)

for w in pre_nouns:
    nouns.append(w[0])

count = Counter(nouns)

pprint(count)

f = codecs.open('wordclouds/서울/' + filename+"_wordCounting.txt", 'w', encoding='utf-8')
f.write(str(count))

tagbox = count.most_common(40)
taglist = pytagcloud.make_tags(tagbox, maxsize=80)

pytagcloud.create_tag_image(taglist, 'wordclouds/서울/' + filename + '.jpg', size=(900, 600), fontname='korean', rectangular=False)

class MyTokenizer:
    def __init__(self, tagger):
        self.tagger = tagger
    def __call__(self, sent):
        pos = self.tagger.pos(sent)
        pos = ['{}/{}'.format(word,tag) for word, tag in pos]
        return pos


filename = '서울/종로구(2019)'

with open('outputs/' + filename + '.txt', 'r') as ft:
    text = ft.readlines()


my_tokenizer = MyTokenizer(postprocessor)
vectorizer = CountVectorizer(tokenizer = my_tokenizer)
X = vectorizer.fit_transform(text)

Xc = (X.T * X)
Xc.setdiag(0)
print(Xc.todense())

# pandas 를 사용하여 gephi 에서 보여줄 word-word co-occurrence matrix csv 파일을 생성
names = vectorizer.get_feature_names() # 엔티티 이름들을 보여준다
df = pd.DataFrame(data = Xc.toarray(), columns = names, index = names)
df.to_csv('gephi/' + filename + '.csv', sep = ',')

id2word = corpora.Dictionary(token)
totalCorpus = token

## 출현단어 2개 이하는 무시하고 dictionary 생성

min_count = 3
word_counter = Counter((word for words in token for word in words))
removal_word_idxs = {
    id2word.token2id[word] for word, count in word_counter.items()
    if count < min_count
}

id2word.filter_tokens(removal_word_idxs)
id2word.compactify()
print('dictionary size : %d' % len(id2word))

## Term - Document Matrix 생성
corpus = [id2word.doc2bow(text) for text in totalCorpus]

"""
lda_model = models.ldamodel.LdaModel(corpus=corpus,
id2word=id2word,
num_topics=20,
random_state=100,
update_every=1,
chunksize=100,
passes=10,
alpha='auto',
per_word_topics=True)
"""

# mallet lda 적용
mallet_path = 'source/mallet-2.0.8/bin/mallet'
lda_model = models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=20, id2word=id2word)

pprint(lda_model.print_topics())

# Compute Coherence Score
coherence_model_lda = models.CoherenceModel(model=lda_model, texts=totalCorpus, dictionary=id2word, coherence='c_v')
coherence_lda = coherence_model_lda.get_coherence()
print('\nCoherence Score: ', coherence_lda)