386 lines
15 KiB
Python
386 lines
15 KiB
Python
|
"""
|
||
|
|
===========================================
|
||
|
|
FeatureHasher and DictVectorizer Comparison
|
||
|
|
===========================================
|
||
|
|
|
||
|
|
In this example we illustrate text vectorization, which is the process of
|
||
|
|
representing non-numerical input data (such as dictionaries or text documents)
|
||
|
|
as vectors of real numbers.
|
||
|
|
|
||
|
|
We first compare :func:`~sklearn.feature_extraction.FeatureHasher` and
|
||
|
|
:func:`~sklearn.feature_extraction.DictVectorizer` by using both methods to
|
||
|
|
vectorize text documents that are preprocessed (tokenized) with the help of a
|
||
|
|
custom Python function.
|
||
|
|
|
||
|
|
Later we introduce and analyze the text-specific vectorizers
|
||
|
|
:func:`~sklearn.feature_extraction.text.HashingVectorizer`,
|
||
|
|
:func:`~sklearn.feature_extraction.text.CountVectorizer` and
|
||
|
|
:func:`~sklearn.feature_extraction.text.TfidfVectorizer` that handle both the
|
||
|
|
tokenization and the assembling of the feature matrix within a single class.
|
||
|
|
|
||
|
|
The objective of the example is to demonstrate the usage of text vectorization
|
||
|
|
API and to compare their processing time. See the example scripts
|
||
|
|
:ref:`sphx_glr_auto_examples_text_plot_document_classification_20newsgroups.py`
|
||
|
|
and :ref:`sphx_glr_auto_examples_text_plot_document_clustering.py` for actual
|
||
|
|
learning on text documents.
|
||
|
|
|
||
|
|
"""
|
||
|
|
|
||
|
|
# Author: Lars Buitinck
|
||
|
|
# Olivier Grisel <olivier.grisel@ensta.org>
|
||
|
|
# Arturo Amor <david-arturo.amor-quiroz@inria.fr>
|
||
|
|
# License: BSD 3 clause
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# Load Data
|
||
|
|
# ---------
|
||
|
|
#
|
||
|
|
# We load data from :ref:`20newsgroups_dataset`, which comprises around
|
||
|
|
# 18000 newsgroups posts on 20 topics split in two subsets: one for training and
|
||
|
|
# one for testing. For the sake of simplicity and reducing the computational
|
||
|
|
# cost, we select a subset of 7 topics and use the training set only.
|
||
|
|
|
||
|
|
from sklearn.datasets import fetch_20newsgroups
|
||
|
|
|
||
|
|
categories = [
|
||
|
|
"alt.atheism",
|
||
|
|
"comp.graphics",
|
||
|
|
"comp.sys.ibm.pc.hardware",
|
||
|
|
"misc.forsale",
|
||
|
|
"rec.autos",
|
||
|
|
"sci.space",
|
||
|
|
"talk.religion.misc",
|
||
|
|
]
|
||
|
|
|
||
|
|
print("Loading 20 newsgroups training data")
|
||
|
|
raw_data, _ = fetch_20newsgroups(subset="train", categories=categories, return_X_y=True)
|
||
|
|
data_size_mb = sum(len(s.encode("utf-8")) for s in raw_data) / 1e6
|
||
|
|
print(f"{len(raw_data)} documents - {data_size_mb:.3f}MB")
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# Define preprocessing functions
|
||
|
|
# ------------------------------
|
||
|
|
#
|
||
|
|
# A token may be a word, part of a word or anything comprised between spaces or
|
||
|
|
# symbols in a string. Here we define a function that extracts the tokens using
|
||
|
|
# a simple regular expression (regex) that matches Unicode word characters. This
|
||
|
|
# includes most characters that can be part of a word in any language, as well
|
||
|
|
# as numbers and the underscore:
|
||
|
|
|
||
|
|
import re
|
||
|
|
|
||
|
|
|
||
|
|
def tokenize(doc):
|
||
|
|
"""Extract tokens from doc.
|
||
|
|
|
||
|
|
This uses a simple regex that matches word characters to break strings
|
||
|
|
into tokens. For a more principled approach, see CountVectorizer or
|
||
|
|
TfidfVectorizer.
|
||
|
|
"""
|
||
|
|
return (tok.lower() for tok in re.findall(r"\w+", doc))
|
||
|
|
|
||
|
|
|
||
|
|
list(tokenize("This is a simple example, isn't it?"))
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# We define an additional function that counts the (frequency of) occurrence of
|
||
|
|
# each token in a given document. It returns a frequency dictionary to be used
|
||
|
|
# by the vectorizers.
|
||
|
|
|
||
|
|
from collections import defaultdict
|
||
|
|
|
||
|
|
|
||
|
|
def token_freqs(doc):
|
||
|
|
"""Extract a dict mapping tokens from doc to their occurrences."""
|
||
|
|
|
||
|
|
freq = defaultdict(int)
|
||
|
|
for tok in tokenize(doc):
|
||
|
|
freq[tok] += 1
|
||
|
|
return freq
|
||
|
|
|
||
|
|
|
||
|
|
token_freqs("That is one example, but this is another one")
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# Observe in particular that the repeated token `"is"` is counted twice for
|
||
|
|
# instance.
|
||
|
|
#
|
||
|
|
# Breaking a text document into word tokens, potentially losing the order
|
||
|
|
# information between the words in a sentence is often called a `Bag of Words
|
||
|
|
# representation <https://en.wikipedia.org/wiki/Bag-of-words_model>`_.
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# DictVectorizer
|
||
|
|
# --------------
|
||
|
|
#
|
||
|
|
# First we benchmark the :func:`~sklearn.feature_extraction.DictVectorizer`,
|
||
|
|
# then we compare it to :func:`~sklearn.feature_extraction.FeatureHasher` as
|
||
|
|
# both of them receive dictionaries as input.
|
||
|
|
|
||
|
|
from time import time
|
||
|
|
|
||
|
|
from sklearn.feature_extraction import DictVectorizer
|
||
|
|
|
||
|
|
dict_count_vectorizers = defaultdict(list)
|
||
|
|
|
||
|
|
t0 = time()
|
||
|
|
vectorizer = DictVectorizer()
|
||
|
|
vectorizer.fit_transform(token_freqs(d) for d in raw_data)
|
||
|
|
duration = time() - t0
|
||
|
|
dict_count_vectorizers["vectorizer"].append(
|
||
|
|
vectorizer.__class__.__name__ + "\non freq dicts"
|
||
|
|
)
|
||
|
|
dict_count_vectorizers["speed"].append(data_size_mb / duration)
|
||
|
|
print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s")
|
||
|
|
print(f"Found {len(vectorizer.get_feature_names_out())} unique terms")
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# The actual mapping from text token to column index is explicitly stored in
|
||
|
|
# the `.vocabulary_` attribute which is a potentially very large Python
|
||
|
|
# dictionary:
|
||
|
|
type(vectorizer.vocabulary_)
|
||
|
|
|
||
|
|
# %%
|
||
|
|
len(vectorizer.vocabulary_)
|
||
|
|
|
||
|
|
# %%
|
||
|
|
vectorizer.vocabulary_["example"]
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# FeatureHasher
|
||
|
|
# -------------
|
||
|
|
#
|
||
|
|
# Dictionaries take up a large amount of storage space and grow in size as the
|
||
|
|
# training set grows. Instead of growing the vectors along with a dictionary,
|
||
|
|
# feature hashing builds a vector of pre-defined length by applying a hash
|
||
|
|
# function `h` to the features (e.g., tokens), then using the hash values
|
||
|
|
# directly as feature indices and updating the resulting vector at those
|
||
|
|
# indices. When the feature space is not large enough, hashing functions tend to
|
||
|
|
# map distinct values to the same hash code (hash collisions). As a result, it
|
||
|
|
# is impossible to determine what object generated any particular hash code.
|
||
|
|
#
|
||
|
|
# Because of the above it is impossible to recover the original tokens from the
|
||
|
|
# feature matrix and the best approach to estimate the number of unique terms in
|
||
|
|
# the original dictionary is to count the number of active columns in the
|
||
|
|
# encoded feature matrix. For such a purpose we define the following function:
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
|
||
|
|
|
||
|
|
def n_nonzero_columns(X):
|
||
|
|
"""Number of columns with at least one non-zero value in a CSR matrix.
|
||
|
|
|
||
|
|
This is useful to count the number of features columns that are effectively
|
||
|
|
active when using the FeatureHasher.
|
||
|
|
"""
|
||
|
|
return len(np.unique(X.nonzero()[1]))
|
||
|
|
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# The default number of features for the
|
||
|
|
# :func:`~sklearn.feature_extraction.FeatureHasher` is 2**20. Here we set
|
||
|
|
# `n_features = 2**18` to illustrate hash collisions.
|
||
|
|
#
|
||
|
|
# **FeatureHasher on frequency dictionaries**
|
||
|
|
|
||
|
|
from sklearn.feature_extraction import FeatureHasher
|
||
|
|
|
||
|
|
t0 = time()
|
||
|
|
hasher = FeatureHasher(n_features=2**18)
|
||
|
|
X = hasher.transform(token_freqs(d) for d in raw_data)
|
||
|
|
duration = time() - t0
|
||
|
|
dict_count_vectorizers["vectorizer"].append(
|
||
|
|
hasher.__class__.__name__ + "\non freq dicts"
|
||
|
|
)
|
||
|
|
dict_count_vectorizers["speed"].append(data_size_mb / duration)
|
||
|
|
print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s")
|
||
|
|
print(f"Found {n_nonzero_columns(X)} unique tokens")
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# The number of unique tokens when using the
|
||
|
|
# :func:`~sklearn.feature_extraction.FeatureHasher` is lower than those obtained
|
||
|
|
# using the :func:`~sklearn.feature_extraction.DictVectorizer`. This is due to
|
||
|
|
# hash collisions.
|
||
|
|
#
|
||
|
|
# The number of collisions can be reduced by increasing the feature space.
|
||
|
|
# Notice that the speed of the vectorizer does not change significantly when
|
||
|
|
# setting a large number of features, though it causes larger coefficient
|
||
|
|
# dimensions and then requires more memory usage to store them, even if a
|
||
|
|
# majority of them is inactive.
|
||
|
|
|
||
|
|
t0 = time()
|
||
|
|
hasher = FeatureHasher(n_features=2**22)
|
||
|
|
X = hasher.transform(token_freqs(d) for d in raw_data)
|
||
|
|
duration = time() - t0
|
||
|
|
|
||
|
|
print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s")
|
||
|
|
print(f"Found {n_nonzero_columns(X)} unique tokens")
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# We confirm that the number of unique tokens gets closer to the number of
|
||
|
|
# unique terms found by the :func:`~sklearn.feature_extraction.DictVectorizer`.
|
||
|
|
#
|
||
|
|
# **FeatureHasher on raw tokens**
|
||
|
|
#
|
||
|
|
# Alternatively, one can set `input_type="string"` in the
|
||
|
|
# :func:`~sklearn.feature_extraction.FeatureHasher` to vectorize the strings
|
||
|
|
# output directly from the customized `tokenize` function. This is equivalent to
|
||
|
|
# passing a dictionary with an implied frequency of 1 for each feature name.
|
||
|
|
|
||
|
|
t0 = time()
|
||
|
|
hasher = FeatureHasher(n_features=2**18, input_type="string")
|
||
|
|
X = hasher.transform(tokenize(d) for d in raw_data)
|
||
|
|
duration = time() - t0
|
||
|
|
dict_count_vectorizers["vectorizer"].append(
|
||
|
|
hasher.__class__.__name__ + "\non raw tokens"
|
||
|
|
)
|
||
|
|
dict_count_vectorizers["speed"].append(data_size_mb / duration)
|
||
|
|
print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s")
|
||
|
|
print(f"Found {n_nonzero_columns(X)} unique tokens")
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# We now plot the speed of the above methods for vectorizing.
|
||
|
|
|
||
|
|
import matplotlib.pyplot as plt
|
||
|
|
|
||
|
|
fig, ax = plt.subplots(figsize=(12, 6))
|
||
|
|
|
||
|
|
y_pos = np.arange(len(dict_count_vectorizers["vectorizer"]))
|
||
|
|
ax.barh(y_pos, dict_count_vectorizers["speed"], align="center")
|
||
|
|
ax.set_yticks(y_pos)
|
||
|
|
ax.set_yticklabels(dict_count_vectorizers["vectorizer"])
|
||
|
|
ax.invert_yaxis()
|
||
|
|
_ = ax.set_xlabel("speed (MB/s)")
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# In both cases :func:`~sklearn.feature_extraction.FeatureHasher` is
|
||
|
|
# approximately twice as fast as
|
||
|
|
# :func:`~sklearn.feature_extraction.DictVectorizer`. This is handy when dealing
|
||
|
|
# with large amounts of data, with the downside of losing the invertibility of
|
||
|
|
# the transformation, which in turn makes the interpretation of a model a more
|
||
|
|
# complex task.
|
||
|
|
#
|
||
|
|
# The `FeatureHeasher` with `input_type="string"` is slightly faster than the
|
||
|
|
# variant that works on frequency dict because it does not count repeated
|
||
|
|
# tokens: each token is implicitly counted once, even if it was repeated.
|
||
|
|
# Depending on the downstream machine learning task, it can be a limitation or
|
||
|
|
# not.
|
||
|
|
#
|
||
|
|
# Comparison with special purpose text vectorizers
|
||
|
|
# ------------------------------------------------
|
||
|
|
#
|
||
|
|
# :func:`~sklearn.feature_extraction.text.CountVectorizer` accepts raw data as
|
||
|
|
# it internally implements tokenization and occurrence counting. It is similar
|
||
|
|
# to the :func:`~sklearn.feature_extraction.DictVectorizer` when used along with
|
||
|
|
# the customized function `token_freqs` as done in the previous section. The
|
||
|
|
# difference being that :func:`~sklearn.feature_extraction.text.CountVectorizer`
|
||
|
|
# is more flexible. In particular it accepts various regex patterns through the
|
||
|
|
# `token_pattern` parameter.
|
||
|
|
|
||
|
|
from sklearn.feature_extraction.text import CountVectorizer
|
||
|
|
|
||
|
|
t0 = time()
|
||
|
|
vectorizer = CountVectorizer()
|
||
|
|
vectorizer.fit_transform(raw_data)
|
||
|
|
duration = time() - t0
|
||
|
|
dict_count_vectorizers["vectorizer"].append(vectorizer.__class__.__name__)
|
||
|
|
dict_count_vectorizers["speed"].append(data_size_mb / duration)
|
||
|
|
print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s")
|
||
|
|
print(f"Found {len(vectorizer.get_feature_names_out())} unique terms")
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# We see that using the :func:`~sklearn.feature_extraction.text.CountVectorizer`
|
||
|
|
# implementation is approximately twice as fast as using the
|
||
|
|
# :func:`~sklearn.feature_extraction.DictVectorizer` along with the simple
|
||
|
|
# function we defined for mapping the tokens. The reason is that
|
||
|
|
# :func:`~sklearn.feature_extraction.text.CountVectorizer` is optimized by
|
||
|
|
# reusing a compiled regular expression for the full training set instead of
|
||
|
|
# creating one per document as done in our naive tokenize function.
|
||
|
|
#
|
||
|
|
# Now we make a similar experiment with the
|
||
|
|
# :func:`~sklearn.feature_extraction.text.HashingVectorizer`, which is
|
||
|
|
# equivalent to combining the "hashing trick" implemented by the
|
||
|
|
# :func:`~sklearn.feature_extraction.FeatureHasher` class and the text
|
||
|
|
# preprocessing and tokenization of the
|
||
|
|
# :func:`~sklearn.feature_extraction.text.CountVectorizer`.
|
||
|
|
|
||
|
|
from sklearn.feature_extraction.text import HashingVectorizer
|
||
|
|
|
||
|
|
t0 = time()
|
||
|
|
vectorizer = HashingVectorizer(n_features=2**18)
|
||
|
|
vectorizer.fit_transform(raw_data)
|
||
|
|
duration = time() - t0
|
||
|
|
dict_count_vectorizers["vectorizer"].append(vectorizer.__class__.__name__)
|
||
|
|
dict_count_vectorizers["speed"].append(data_size_mb / duration)
|
||
|
|
print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s")
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# We can observe that this is the fastest text tokenization strategy so far,
|
||
|
|
# assuming that the downstream machine learning task can tolerate a few
|
||
|
|
# collisions.
|
||
|
|
#
|
||
|
|
# TfidfVectorizer
|
||
|
|
# ---------------
|
||
|
|
#
|
||
|
|
# In a large text corpus, some words appear with higher frequency (e.g. "the",
|
||
|
|
# "a", "is" in English) and do not carry meaningful information about the actual
|
||
|
|
# contents of a document. If we were to feed the word count data directly to a
|
||
|
|
# classifier, those very common terms would shadow the frequencies of rarer yet
|
||
|
|
# more informative terms. In order to re-weight the count features into floating
|
||
|
|
# point values suitable for usage by a classifier it is very common to use the
|
||
|
|
# tf-idf transform as implemented by the
|
||
|
|
# :func:`~sklearn.feature_extraction.text.TfidfTransformer`. TF stands for
|
||
|
|
# "term-frequency" while "tf-idf" means term-frequency times inverse
|
||
|
|
# document-frequency.
|
||
|
|
#
|
||
|
|
# We now benchmark the :func:`~sklearn.feature_extraction.text.TfidfVectorizer`,
|
||
|
|
# which is equivalent to combining the tokenization and occurrence counting of
|
||
|
|
# the :func:`~sklearn.feature_extraction.text.CountVectorizer` along with the
|
||
|
|
# normalizing and weighting from a
|
||
|
|
# :func:`~sklearn.feature_extraction.text.TfidfTransformer`.
|
||
|
|
|
||
|
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
||
|
|
|
||
|
|
t0 = time()
|
||
|
|
vectorizer = TfidfVectorizer()
|
||
|
|
vectorizer.fit_transform(raw_data)
|
||
|
|
duration = time() - t0
|
||
|
|
dict_count_vectorizers["vectorizer"].append(vectorizer.__class__.__name__)
|
||
|
|
dict_count_vectorizers["speed"].append(data_size_mb / duration)
|
||
|
|
print(f"done in {duration:.3f} s at {data_size_mb / duration:.1f} MB/s")
|
||
|
|
print(f"Found {len(vectorizer.get_feature_names_out())} unique terms")
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# Summary
|
||
|
|
# -------
|
||
|
|
# Let's conclude this notebook by summarizing all the recorded processing speeds
|
||
|
|
# in a single plot:
|
||
|
|
|
||
|
|
fig, ax = plt.subplots(figsize=(12, 6))
|
||
|
|
|
||
|
|
y_pos = np.arange(len(dict_count_vectorizers["vectorizer"]))
|
||
|
|
ax.barh(y_pos, dict_count_vectorizers["speed"], align="center")
|
||
|
|
ax.set_yticks(y_pos)
|
||
|
|
ax.set_yticklabels(dict_count_vectorizers["vectorizer"])
|
||
|
|
ax.invert_yaxis()
|
||
|
|
_ = ax.set_xlabel("speed (MB/s)")
|
||
|
|
|
||
|
|
# %%
|
||
|
|
# Notice from the plot that
|
||
|
|
# :func:`~sklearn.feature_extraction.text.TfidfVectorizer` is slightly slower
|
||
|
|
# than :func:`~sklearn.feature_extraction.text.CountVectorizer` because of the
|
||
|
|
# extra operation induced by the
|
||
|
|
# :func:`~sklearn.feature_extraction.text.TfidfTransformer`.
|
||
|
|
#
|
||
|
|
# Also notice that, by setting the number of features `n_features = 2**18`, the
|
||
|
|
# :func:`~sklearn.feature_extraction.text.HashingVectorizer` performs better
|
||
|
|
# than the :func:`~sklearn.feature_extraction.text.CountVectorizer` at the
|
||
|
|
# expense of inversibility of the transformation due to hash collisions.
|
||
|
|
#
|
||
|
|
# We highlight that :func:`~sklearn.feature_extraction.text.CountVectorizer` and
|
||
|
|
# :func:`~sklearn.feature_extraction.text.HashingVectorizer` perform better than
|
||
|
|
# their equivalent :func:`~sklearn.feature_extraction.DictVectorizer` and
|
||
|
|
# :func:`~sklearn.feature_extraction.FeatureHasher` on manually tokenized
|
||
|
|
# documents since the internal tokenization step of the former vectorizers
|
||
|
|
# compiles a regular expression once and then reuses it for all the documents.
|