TF-IDF from Scratch

บทความโดย อ.ดร.ณัฐโชติ พรหมฤทธิ์
ภาควิชาคอมพิวเตอร์
คณะวิทยาศาสตร์
มหาวิทยาลัยศิลปากร

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer

documentA = 'The cat sat on my face'
documentB = 'The dog sat on my bed'

bagOfWordsA = documentA.split(' ')
bagOfWordsB = documentB.split(' ')

bagOfWordsA

bagOfWordsB

uniqueWords = set(bagOfWordsA).union(set(bagOfWordsB))

uniqueWords

numOfWordsA = dict.fromkeys(uniqueWords, 0)

numOfWordsA

for word in bagOfWordsA:
    numOfWordsA[word] += 1

numOfWordsA

numOfWordsB = dict.fromkeys(uniqueWords, 0)

numOfWordsB

for word in bagOfWordsB:
    numOfWordsB[word] += 1

numOfWordsB

def computeTF(wordDict, bagOfWords):
    tfDict = {}
    bagOfWordsCount = len(bagOfWords)
    for word, count in wordDict.items():
        tfDict[word] = count / float(bagOfWordsCount)
    return tfDict

tfA = computeTF(numOfWordsA, bagOfWordsA)
tfB = computeTF(numOfWordsB, bagOfWordsB)

tfA

tfB

def computeIDF(documents):
    import math
    N = len(documents)
    
    idfDict = dict.fromkeys(documents[0].keys(), 0)
    for document in documents:
        for word, val in document.items():
            if val > 0:
                idfDict[word] += 1
    
    for word, val in idfDict.items():
        idfDict[word] = math.log(N / float(val), 10)
    return idfDict

import math

1/6*math.log(2/1, 10)

idfs = computeIDF([numOfWordsA, numOfWordsB])

idfs

def computeTFIDF(tfBagOfWords, idfs):
    tfidf = {}
    for word, val in tfBagOfWords.items():
        tfidf[word] = val * idfs[word]
    return tfidf

tfidfA = computeTFIDF(tfA, idfs)
tfidfB = computeTFIDF(tfB, idfs)

df = pd.DataFrame([tfidfA, tfidfB])
df

vectorizer = TfidfVectorizer()
vectors = vectorizer.fit_transform([documentA, documentB])

feature_names = vectorizer.get_feature_names()

feature_names

dense = vectors.todense()

denselist = dense.tolist()

df = pd.DataFrame(denselist, columns=feature_names)
df