Introduction to Word Embeddings

%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
%config InlineBackend.figure_format = 'retina'
!pip install -U torchtext
import torch
import torchtext
from torchtext.vocab import GloVe
import umap
pd.options.display.max_colwidth=-1
#

glove = GloVe(name = '840B', dim = 300)

def get_word_vec(word):
  assert word in glove.stoi, f"{word} is OOV!"
  vec = glove.vectors[glove.stoi[word]]
  return vec


def cosine_similarity(word1, word2):
  vec1 = get_word_vec(word1)
  vec2 = get_word_vec(word2)
  return torch.cosine_similarity(vec1.unsqueeze(0),
                                 vec2.unsqueeze(0)).item()


def get_nearest_neighbours(word_or_vec, n=10):
    vec = None
    if isinstance(word_or_vec, str):
        vec = get_word_vec(word_or_vec)
    else:
        vec = word_or_vec
        
    topn =  torch.argsort(torch.mv(glove.vectors, vec), descending=True)[:n]
        
    return [glove.itos[idx] for idx in topn]

def plot_embeddings(words):
  fig, axes = plt.subplots(nrows=len(words), 
                           ncols=1, 
                           sharex=True, 
                           figsize=(20,len(words)))

  for ax, w in zip(axes.flat, words):
    im = ax.imshow(glove.vectors[glove.stoi[w]].view(1,-1), 
                   cmap=plt.cm.coolwarm)
    ax.set_aspect('auto')

  fig.subplots_adjust(right=0.8)
  cbar_ax = fig.add_axes([0.85, 0.15, 0.01, 0.7])
  fig.colorbar(im, cax=cbar_ax)
#

words = ['gurgaon','paratha','dosa', 'pizza', 'pasta']
plot_embeddings(words)
#

E = torch.stack([get_word_vec(w) for w in words], dim=0)

from sklearn.metrics.pairwise import cosine_similarity as mat_cosine_similarity

csim = mat_cosine_similarity(E)
csim.round(3)
#

foods = ['chapati','chana','bhatura','chole','dal',
         'poha','idli','upma','dosa','puri',
         'paratha','jalebi','kachori','khichdi','kofta']

places = ['gurgaon','faridabad','lucknow','noida','delhi',
          'koramangala','bangalore','goa','panaji','coorg',
          'ladakh','amritsar','hyderabad','mumbai','varanasi'
         ]

food_places_df = pd.DataFrame(data={'word' : foods + places, 
                                    'type' : ['food'] * len(foods) + 
                                             ['place'] * len(places)})
E = torch.stack([get_word_vec(row.word) for _,row in food_places_df.iterrows()], 
                 dim=0)

fit = umap.UMAP(n_neighbors=5, random_state=42, metric='cosine')
u = fit.fit_transform(E)

fig, ax = plt.subplots(figsize=(12,4))
sns.scatterplot(u[:,0], u[:,1], ax=ax, hue=food_places_df.type, s=100);
#

num_points = 10000
indices = np.random.choice(glove.vectors.shape[0],size=num_points, 
                           replace=False)

E = glove.vectors[indices]
fit = umap.UMAP(n_neighbors=10, random_state=42, metric='cosine')
u = fit.fit_transform(E)
fig, ax = plt.subplots(figsize=(18,8))
sns.scatterplot(u[:,0], u[:,1], ax=ax, s=10);
#

gender_pairs = [('she','he'),
                ('her','his'),
                ('woman','man'),
                ('daughter','son'),
                ('mother','father'),
               ]
words = [el for pair in gender_pairs for el in pair]
gender_df = pd.DataFrame(data={'gender' : ['female', 'male'] * len(gender_pairs),
                               'word' : words
                              })
X = torch.stack([get_word_vec(w) for w in gender_df.word], dim=0)

from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

pipe = Pipeline([('scaler',StandardScaler()),
                 ('pca', PCA(n_components=2))
                ])

X_reduced = pipe.fit_transform(X)

fig, ax = plt.subplots(figsize=(12,4))

sns.scatterplot(X_reduced[:,0], X_reduced[:,1], ax=ax, hue=gender_df.gender, s=60)

for i, word in enumerate(words):
    ax.annotate(word, (X_reduced[i,0], X_reduced[i,1]))
    
for i in range(0, X_reduced.shape[0], 2):
    ax.plot(X_reduced[i:i+2,0], X_reduced[i:i+2,1], c='gray', linewidth=1)
#

vecs = []
for i in range(0, X.shape[0], 2):
    a = X[i,:]
    b = X[i+1,:]
    mean_vec = (a+b)/2
    vecs.append(a-mean_vec)
    vecs.append(b-mean_vec)
X1 = torch.stack(vecs, dim=0)

vecs = []
for i in range(0, X1.shape[0], 2):
    a = X[i,:]
    b = X[i+1,:]
    vecs.append(b-a)
X2 = torch.stack(vecs, dim=0)

to_masculine_vector = X2.mean(dim=0)

def get_masculine_word(feminine_word):
  word_vector = get_word_vec(feminine_word)
  candidates = get_nearest_neighbours(word_vector + to_masculine_vector)
  return [c for c in candidates if c not in [feminine_word, feminine_word+"s"]][0]
#