In [1]:
import collections
import math
import os
import random
from tempfile import gettempdir
import zipfile

import numpy as np
from six.moves import urllib
from six.moves import xrange  # pylint: disable=redefined-builtin
import tensorflow as tf
In [2]:
# Step 1: Read data
filename = 'text8.zip'

def read_data(filename):
    """Extract the first file enclosed in a zip file as a list of words."""
    with zipfile.ZipFile(filename) as f:
        data = tf.compat.as_str(f.read(f.namelist()[0])).split()
    return data

vocabulary = read_data(filename)
print('Data size', len(vocabulary))
('Data size', 17005207)
In [3]:
# Step 2: Build the dictionary and replace rare words with UNK token.
vocabulary_size = 50000

def build_dataset(words, n_words):
    """Process raw inputs into a dataset."""
    count = [['UNK', -1]]
    count.extend(collections.Counter(words).most_common(n_words - 1))
    dictionary = dict()
    for word, _ in count:
        dictionary[word] = len(dictionary)
    data = list()
    unk_count = 0
    for word in words:
        index = dictionary.get(word, 0)
        if index == 0: 
            unk_count += 1
        data.append(index)
    count[0][1] = unk_count
    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
    return data, count, dictionary, reversed_dictionary

# Filling 4 global variables:
# data - list of codes (integers from 0 to vocabulary_size-1).
#   This is the original text but words are replaced by their codes
# count - map of words(strings) to count of occurrences
# dictionary - map of words(strings) to their codes(integers)
# reverse_dictionary - maps codes(integers) to words(strings)
data, count, dictionary, reverse_dictionary = build_dataset(vocabulary, vocabulary_size)
del vocabulary  # Hint to reduce memory.
print('Most common words (+UNK)', count[:5])
print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])
('Most common words (+UNK)', [['UNK', 418391], ('the', 1061396), ('of', 593677), ('and', 416629), ('one', 411764)])
('Sample data', [5239, 3084, 12, 6, 195, 2, 3137, 46, 59, 156], ['anarchism', 'originated', 'as', 'a', 'term', 'of', 'abuse', 'first', 'used', 'against'])
In [4]:
# Step 3: Function to generate a training batch for the skip-gram model.
data_index = 0
def generate_batch(batch_size, num_skips, skip_window):
    global data_index
    assert batch_size % num_skips == 0
    assert num_skips <= 2 * skip_window
    batch = np.ndarray(shape=(batch_size), dtype=np.int32)
    labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
    span = 2 * skip_window + 1 # [ skip_window target skip_window ]
    buffer = collections.deque(maxlen=span)
    if data_index + span > len(data):
        data_index = 0
    buffer.extend(data[data_index:data_index + span])
    data_index += span
    for i in range(batch_size // num_skips):
        context_words = [w for w in range(span) if w != skip_window]
        words_to_use = random.sample(context_words, num_skips)
        for j, context_word in enumerate(words_to_use):
            batch[i * num_skips + j] = buffer[skip_window]
            labels[i * num_skips + j, 0] = buffer[context_word]
            
        if data_index == len(data):
            buffer[:] = data[:span]
            data_index = span
        else:
            buffer.append(data[data_index])
            data_index += 1
        if data_index == len(data):
            buffer[:] = data[:span]
            data_index += 1
    data_index = (data_index + len(data) - span) % len(data)
    return batch, labels

batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
for i in range(8):
    print(batch[i], reverse_dictionary[batch[i]],
        '->', labels[i, 0], reverse_dictionary[labels[i, 0]])
(3084, 'originated', '->', 12, 'as')
(3084, 'originated', '->', 5239, 'anarchism')
(12, 'as', '->', 3084, 'originated')
(12, 'as', '->', 6, 'a')
(6, 'a', '->', 195, 'term')
(6, 'a', '->', 12, 'as')
(195, 'term', '->', 2, 'of')
(195, 'term', '->', 6, 'a')
In [5]:
# Step 4: Build and train a skip-gram model.

batch_size = 128
embedding_size = 128  # Dimension of the embedding vector.
skip_window = 1       # How many words to consider left and right.
num_skips = 2         # How many times to reuse an input to generate a label.
num_sampled = 64      # Number of negative examples to sample.

# We pick a random validation set to sample nearest neighbors. Here we limit the
# validation samples to the words that have a low numeric ID, which by
# construction are also the most frequent. These 3 variables are used only for
# displaying model accuracy, they don't affect calculation.
valid_size = 16     # Random set of words to evaluate similarity on.
valid_window = 100  # Only pick dev samples in the head of the distribution.
valid_examples = np.random.choice(valid_window, valid_size, replace=False)


graph = tf.Graph()

with graph.as_default():
    # Input data.
    train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
    train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
    valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
    
    # Ops and variables pinned to the CPU because of missing GPU implementation
    with tf.device('/cpu:0'):
        # Look up embeddings for inputs.
        embeddings = tf.Variable(
            tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
        embed = tf.nn.embedding_lookup(embeddings, train_inputs) # 根据train_inputs选出embeddings中的对应行
        
        # Construct the variables for the NCE loss
        nce_weights = tf.Variable(
            tf.truncated_normal([vocabulary_size, embedding_size], 
                                stddev=1.0 / math.sqrt(embedding_size)))
        nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
        
    # Compute the average NCE loss for the batch.
    loss = tf.reduce_mean(
        tf.nn.nce_loss(weights=nce_weights,
                       biases=nce_biases,
                       labels=train_labels,
                       inputs=embed,
                       num_sampled=num_sampled,
                       num_classes=vocabulary_size))
    
    # Construct the SGD optimizer using a learning rate of 1.0.
    optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
    
    # Compute the cosine similarity between minibatch examples and all embeddings.
    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
    normalized_embeddings = embeddings / norm
    valid_embeddings = tf.nn.embedding_lookup(
        normalized_embeddings, valid_dataset)
    similarity = tf.matmul(
        valid_embeddings, normalized_embeddings, transpose_b=True)
    
    # Add variable initializer.
    init = tf.global_variables_initializer()
In [6]:
# Step 5: Begin training.
num_steps = 100001

with tf.Session(graph=graph) as session:
    init.run()
    print 'Initialized'
    
    average_loss = 0
    for step in xrange(num_steps):
        batch_inputs, batch_labels = generate_batch(
            batch_size, num_skips, skip_window)
        feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
        
        _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)
        average_loss += loss_val
        
        if step % 2000 == 0:
            if step > 0:
                average_loss /= 2000
            print'Average loss at step ', step, ': ', average_loss
            average_loss = 0
            
        if step % 10000 == 0:
            sim = similarity.eval()
            for i in xrange(valid_size):
                valid_word = reverse_dictionary[valid_examples[i]]
                top_k = 8
                nearest = (-sim[i, :]).argsort()[1:top_k + 1]
                log_str = 'Nearest to %s: ' % valid_word
                for k in xrange(top_k):
                    close_word = reverse_dictionary[nearest[k]]
                    log_str = '%s %s, ' % (log_str, close_word)
                print log_str
    final_embeddings = normalized_embeddings.eval()
Initialized
Average loss at step  0 :  312.81640625
Nearest to d:  jester,  dryness,  animaniacs,  mccann,  wyndham,  stouffer,  load,  bottomed, 
Nearest to s:  lacrosse,  poppers,  ret,  hamath,  planet,  audiobook,  vogue,  camcorder, 
Nearest to as:  mesogens,  persecutions,  patchwork,  danzig,  monogamy,  orgasm,  bulb,  battletech, 
Nearest to states:  mat,  tear,  sof,  meteorological,  carbonates,  professes,  softer,  vagus, 
Nearest to to:  dancers,  userland,  zoning,  rite,  schemas,  stretched,  nixon,  stroustrup, 
Nearest to american:  modular,  homomorphism,  skipping,  vinnie,  rommel,  euphoria,  override,  clarity, 
Nearest to years:  mediation,  pfp,  xxviii,  nfc,  hennig,  dostoyevsky,  cambodia,  hne, 
Nearest to time:  vera,  embroiled,  nonaligned,  woodward,  updates,  markup,  plebs,  cursus, 
Nearest to their:  foundation,  hallucinogen,  falcons,  aerosols,  knut,  schiff,  alaska,  strengths, 
Nearest to of:  fujian,  cliffs,  viral,  dotted,  cashier,  goto,  jem,  booby, 
Nearest to six:  adas,  fluency,  radars,  folkestone,  oppressive,  fiedler,  od,  lentils, 
Nearest to been:  plastics,  declining,  firmly,  graphs,  complaint,  landis,  abbotsford,  arena, 
Nearest to not:  manipulated,  underpinnings,  dentistry,  deontology,  incentives,  gbit,  undisclosed,  silverstein, 
Nearest to eight:  dvo,  enlist,  cystic,  heathrow,  scan,  xliii,  desks,  hardliners, 
Nearest to also:  evergreen,  greenery,  hypothesised,  antiprism,  dsn,  savoury,  rewrite,  trier, 
Nearest to than:  mastiff,  ecu,  turned,  tens,  cow,  housewife,  microgravity,  passions, 
Average loss at step  2000 :  113.300795749
Average loss at step  4000 :  52.7917348313
Average loss at step  6000 :  33.379413782
Average loss at step  8000 :  23.4939553343
Average loss at step  10000 :  17.8640995543
Nearest to d:  palm,  nine,  jester,  dryness,  insulation,  record,  abbey,  feat, 
Nearest to s:  the,  and,  constant,  cheapest,  audiobook,  therapists,  enzyme,  zero, 
Nearest to as:  and,  in,  is,  ada,  k,  var,  archie,  was, 
Nearest to states:  tear,  archie,  newsgroup,  server,  softer,  vikings,  giuseppe,  engineering, 
Nearest to to:  and,  in,  gland,  agenda,  concrete,  heretical,  nine,  by, 
Nearest to american:  baldwin,  and,  filmmakers,  pair,  worms,  sold,  skipping,  queen, 
Nearest to years:  two,  mediation,  six,  nine,  nfc,  cambodia,  challenge,  aberdeenshire, 
Nearest to time:  vera,  supported,  alexandria,  solf,  atemi,  agnostic,  stig,  tasks, 
Nearest to their:  the,  foundation,  unarable,  alaska,  an,  fao,  trial,  extends, 
Nearest to of:  in,  and,  for,  archie,  beers,  with,  fao,  molinari, 
Nearest to six:  nine,  zero,  one,  three,  agave,  four,  survey,  cardinality, 
Nearest to been:  graphs,  phi,  plastics,  timeline,  coke,  pacifist,  climate,  arena, 
Nearest to not:  biscuit,  embedded,  unprecedented,  irrelevant,  works,  it,  agave,  manipulated, 
Nearest to eight:  nine,  zero,  archie,  gland,  agave,  three,  lnot,  four, 
Nearest to also:  agave,  um,  barrow,  UNK,  english,  anything,  whether,  wing, 
Nearest to than:  studd,  turned,  tens,  astounding,  armenians,  little,  passions,  dreyfus, 
Average loss at step  12000 :  14.3712847657
Average loss at step  14000 :  11.8041950456
Average loss at step  16000 :  9.99922999871
Average loss at step  18000 :  8.42966127658
Average loss at step  20000 :  8.02890710914
Nearest to d:  b,  rainforest,  and,  abbey,  agouti,  animaniacs,  operatorname,  dryness, 
Nearest to s:  and,  zero,  agouti,  the,  his,  or,  circ,  of, 
Nearest to as:  and,  is,  agouti,  for,  was,  in,  into,  by, 
Nearest to states:  tear,  circ,  archie,  operatorname,  giuseppe,  antiprism,  agouti,  newsgroup, 
Nearest to to:  and,  heretical,  nine,  in,  for,  can,  not,  concrete, 
Nearest to american:  and,  ambrosia,  homomorphism,  agouti,  baldwin,  worms,  pair,  filmmakers, 
Nearest to years:  mediation,  two,  challenge,  three,  agouti,  six,  nfc,  cambodia, 
Nearest to time:  vera,  supported,  solf,  operatorname,  agnostic,  atemi,  dasyprocta,  embroiled, 
Nearest to their:  the,  his,  its,  gollancz,  this,  circ,  unarable,  alcuin, 
Nearest to of:  and,  in,  for,  nine,  archie,  from,  agouti,  eight, 
Nearest to six:  nine,  eight,  three,  four,  five,  zero,  two,  operatorname, 
Nearest to been:  amazonas,  graphs,  plastics,  also,  by,  phi,  timeline,  pacifist, 
Nearest to not:  to,  it,  biscuit,  agouti,  and,  amp,  embedded,  agave, 
Nearest to eight:  nine,  six,  zero,  five,  four,  three,  seven,  two, 
Nearest to also:  amazonas,  been,  antiprism,  pathway,  agave,  um,  ppm,  barrow, 
Nearest to than:  studd,  turned,  passions,  aisle,  tens,  astounding,  eve,  agouti, 
Average loss at step  22000 :  7.00025532711
Average loss at step  24000 :  6.86389362502
Average loss at step  26000 :  6.69434413671
Average loss at step  28000 :  6.24606887269
Average loss at step  30000 :  5.92552219927
Nearest to d:  b,  rainforest,  and,  otimes,  abbey,  agouti,  irritate,  UNK, 
Nearest to s:  and,  agouti,  two,  his,  zero,  or,  circ,  six, 
Nearest to as:  agouti,  and,  is,  by,  for,  in,  into,  abet, 
Nearest to states:  tear,  trinomial,  circ,  giuseppe,  antiprism,  archie,  operatorname,  akihabara, 
Nearest to to:  heretical,  for,  can,  not,  maximilian,  in,  and,  dasyprocta, 
Nearest to american:  and,  primigenius,  ambrosia,  homomorphism,  otimes,  filmmakers,  worms,  agouti, 
Nearest to years:  mediation,  two,  nfc,  three,  abakan,  six,  challenge,  agouti, 
Nearest to time:  vera,  solf,  operatorname,  agnostic,  supported,  woodward,  atemi,  dasyprocta, 
Nearest to their:  the,  its,  his,  gollancz,  this,  unarable,  circ,  falcons, 
Nearest to of:  and,  in,  for,  archie,  from,  eight,  same,  nine, 
Nearest to six:  eight,  four,  nine,  five,  three,  zero,  seven,  two, 
Nearest to been:  amazonas,  by,  also,  graphs,  phi,  plastics,  pacifist,  be, 
Nearest to not:  to,  it,  agouti,  biscuit,  now,  also,  amp,  embedded, 
Nearest to eight:  nine,  six,  five,  four,  seven,  three,  zero,  two, 
Nearest to also:  now,  amazonas,  which,  been,  agave,  not,  antiprism,  still, 
Nearest to than:  studd,  turned,  passions,  tens,  eve,  astounding,  aisle,  or, 
Average loss at step  32000 :  5.97875713861
Average loss at step  34000 :  5.68047000408
Average loss at step  36000 :  5.80485243893
Average loss at step  38000 :  5.52666420221
Average loss at step  40000 :  5.29902702236
Nearest to d:  b,  UNK,  rainforest,  otimes,  and,  eight,  abbey,  phagocytosis, 
Nearest to s:  zero,  and,  his,  agouti,  two,  circ,  aveiro,  or, 
Nearest to as:  zero,  agouti,  in,  abet,  codeine,  minutemen,  and,  eight, 
Nearest to states:  tear,  trinomial,  giuseppe,  circ,  antiprism,  archie,  vernacular,  zero, 
Nearest to to:  zero,  heretical,  not,  dasyprocta,  nine,  concrete,  can,  gland, 
Nearest to american:  and,  primigenius,  ambrosia,  homomorphism,  pair,  zero,  agouti,  eminent, 
Nearest to years:  mediation,  vma,  philanthropy,  nfc,  abakan,  six,  challenge,  two, 
Nearest to time:  vera,  agnostic,  operatorname,  zero,  solf,  vdc,  dasyprocta,  supported, 
Nearest to their:  his,  its,  the,  zero,  gollancz,  circ,  this,  unarable, 
Nearest to of:  zero,  in,  vma,  and,  for,  archie,  from,  eight, 
Nearest to six:  eight,  four,  three,  five,  seven,  nine,  zero,  two, 
Nearest to been:  be,  also,  by,  was,  amazonas,  graphs,  plastics,  has, 
Nearest to not:  it,  to,  now,  also,  biscuit,  they,  agouti,  zero, 
Nearest to eight:  six,  nine,  seven,  four,  five,  three,  zero,  one, 
Nearest to also:  which,  now,  zero,  still,  amazonas,  been,  not,  agave, 
Nearest to than:  studd,  or,  vdc,  turned,  tens,  passions,  eve,  six, 
Average loss at step  42000 :  5.36294216442
Average loss at step  44000 :  5.28585307002
Average loss at step  46000 :  5.20734661663
Average loss at step  48000 :  5.23575706446
Average loss at step  50000 :  4.98005040586
Nearest to d:  b,  UNK,  rainforest,  otimes,  agouti,  abbey,  operatorname,  c, 
Nearest to s:  his,  and,  agouti,  zero,  of,  the,  four,  circ, 
Nearest to as:  agouti,  in,  for,  by,  eight,  persecutions,  k,  is, 
Nearest to states:  tear,  roshan,  trinomial,  giuseppe,  antiprism,  circ,  archie,  akihabara, 
Nearest to to:  heretical,  and,  maximilian,  kapoor,  dasyprocta,  nine,  concrete,  not, 
Nearest to american:  homomorphism,  primigenius,  and,  ambrosia,  pair,  agouti,  byzantine,  otimes, 
Nearest to years:  mediation,  three,  vma,  nfc,  abakan,  two,  philanthropy,  challenge, 
Nearest to time:  kapoor,  vera,  agnostic,  operatorname,  vdc,  solf,  dasyprocta,  point, 
Nearest to their:  its,  his,  the,  unarable,  posthumous,  circ,  this,  gollancz, 
Nearest to of:  in,  for,  original,  archie,  vma,  from,  eight,  nine, 
Nearest to six:  four,  eight,  three,  five,  seven,  nine,  two,  zero, 
Nearest to been:  be,  was,  also,  graphs,  amazonas,  by,  plastics,  become, 
Nearest to not:  it,  now,  to,  biscuit,  they,  also,  still,  agouti, 
Nearest to eight:  six,  nine,  seven,  five,  four,  three,  zero,  one, 
Nearest to also:  which,  now,  still,  then,  not,  been,  amazonas,  it, 
Nearest to than:  or,  studd,  vdc,  tens,  turned,  yakovlev,  eve,  passions, 
Average loss at step  52000 :  5.04131305182
Average loss at step  54000 :  5.16526106703
Average loss at step  56000 :  5.04458498704
Average loss at step  58000 :  5.04559580553
Average loss at step  60000 :  4.95594383574
Nearest to d:  b,  UNK,  thibetanus,  rainforest,  otimes,  c,  abbey,  prism, 
Nearest to s:  thibetanus,  agouti,  zero,  his,  pulau,  and,  ursus,  circ, 
Nearest to as:  agouti,  in,  ursus,  is,  thibetanus,  for,  by,  persecutions, 
Nearest to states:  roshan,  tear,  trinomial,  circ,  giuseppe,  antiprism,  archie,  operatorname, 
Nearest to to:  heretical,  can,  nine,  pulau,  not,  dasyprocta,  maximilian,  ursus, 
Nearest to american:  pulau,  and,  primigenius,  homomorphism,  ursus,  pair,  british,  byzantine, 
Nearest to years:  mediation,  three,  vma,  pulau,  abakan,  nfc,  six,  philanthropy, 
Nearest to time:  kapoor,  vdc,  operatorname,  thibetanus,  vera,  ursus,  solf,  agnostic, 
Nearest to their:  its,  his,  the,  her,  posthumous,  circ,  pulau,  unarable, 
Nearest to of:  ursus,  archie,  nine,  in,  and,  vma,  for,  original, 
Nearest to six:  four,  eight,  five,  seven,  three,  nine,  zero,  two, 
Nearest to been:  be,  was,  by,  graphs,  become,  amazonas,  also,  ever, 
Nearest to not:  now,  it,  still,  to,  they,  generally,  biscuit,  pulau, 
Nearest to eight:  nine,  six,  seven,  four,  five,  three,  zero,  ursus, 
Nearest to also:  which,  now,  still,  then,  ursus,  amazonas,  it,  often, 
Nearest to than:  or,  studd,  vdc,  mastiff,  tens,  yakovlev,  eve,  cancel, 
Average loss at step  62000 :  5.03237974107
Average loss at step  64000 :  4.85417370009
Average loss at step  66000 :  4.59790955579
Average loss at step  68000 :  4.97592911613
Average loss at step  70000 :  4.89274794972
Nearest to d:  b,  UNK,  cebus,  thibetanus,  rainforest,  c,  otimes,  predicting, 
Nearest to s:  his,  agouti,  thibetanus,  pulau,  thaler,  and,  zero,  ursus, 
Nearest to as:  unassigned,  agouti,  cebus,  capuchin,  microcebus,  ursus,  by,  thibetanus, 
Nearest to states:  roshan,  tear,  trinomial,  giuseppe,  antiprism,  leontopithecus,  circ,  vernacular, 
Nearest to to:  heretical,  can,  not,  would,  maximilian,  dasyprocta,  concrete,  kapoor, 
Nearest to american:  pulau,  primigenius,  homomorphism,  and,  british,  pair,  ursus,  byzantine, 
Nearest to years:  six,  vma,  mediation,  pulau,  abakan,  hennig,  nfc,  philanthropy, 
Nearest to time:  mitral,  kapoor,  vdc,  agnostic,  operatorname,  thibetanus,  vera,  solf, 
Nearest to their:  its,  his,  the,  her,  posthumous,  circ,  pulau,  some, 
Nearest to of:  ursus,  vma,  for,  and,  original,  archie,  cebus,  kapoor, 
Nearest to six:  four,  eight,  five,  three,  seven,  nine,  two,  one, 
Nearest to been:  be,  was,  become,  graphs,  amazonas,  by,  also,  ever, 
Nearest to not:  now,  still,  they,  generally,  it,  to,  pulau,  often, 
Nearest to eight:  six,  seven,  nine,  four,  five,  three,  zero,  two, 
Nearest to also:  now,  still,  which,  often,  then,  typically,  not,  amazonas, 
Nearest to than:  or,  studd,  vdc,  mastiff,  tens,  cancel,  but,  mico, 
Average loss at step  72000 :  4.74016069806
Average loss at step  74000 :  4.80598011744
Average loss at step  76000 :  4.72186035383
Average loss at step  78000 :  4.81422160125
Average loss at step  80000 :  4.81932400197
Nearest to d:  b,  UNK,  cebus,  thibetanus,  c,  rainforest,  l,  predicting, 
Nearest to s:  his,  pulau,  thibetanus,  agouti,  thaler,  circ,  wilkes,  ursus, 
Nearest to as:  unassigned,  agouti,  microcebus,  ursus,  capuchin,  cebus,  thibetanus,  iit, 
Nearest to states:  roshan,  tear,  trinomial,  giuseppe,  antiprism,  circ,  leontopithecus,  vernacular, 
Nearest to to:  heretical,  dasyprocta,  pulau,  maximilian,  would,  can,  for,  ursus, 
Nearest to american:  pulau,  canadian,  british,  homomorphism,  primigenius,  pontificia,  pair,  ursus, 
Nearest to years:  vma,  pulau,  mediation,  abakan,  hennig,  newsreel,  philanthropy,  six, 
Nearest to time:  mitral,  kapoor,  vdc,  operatorname,  thibetanus,  agnostic,  point,  mogul, 
Nearest to their:  its,  his,  the,  her,  pulau,  circ,  posthumous,  some, 
Nearest to of:  ursus,  in,  vma,  archie,  cebus,  agouti,  nine,  for, 
Nearest to six:  four,  five,  seven,  eight,  three,  nine,  two,  zero, 
Nearest to been:  be,  become,  was,  were,  by,  graphs,  ever,  amazonas, 
Nearest to not:  still,  they,  generally,  often,  it,  now,  pulau,  also, 
Nearest to eight:  nine,  seven,  six,  five,  four,  zero,  three,  ursus, 
Nearest to also:  now,  still,  which,  often,  typically,  then,  it,  amazonas, 
Nearest to than:  or,  studd,  vdc,  mastiff,  chronometers,  but,  much,  tens, 
Average loss at step  82000 :  4.78145951879
Average loss at step  84000 :  4.76522578871
Average loss at step  86000 :  4.76914287496
Average loss at step  88000 :  4.74454750943
Average loss at step  90000 :  4.72672532833
Nearest to d:  b,  UNK,  rooks,  cebus,  c,  thibetanus,  l,  predicting, 
Nearest to s:  his,  thibetanus,  agouti,  pulau,  thaler,  ursus,  zero,  and, 
Nearest to as:  unassigned,  agouti,  microcebus,  ursus,  cebus,  by,  thibetanus,  in, 
Nearest to states:  roshan,  trinomial,  antiprism,  giuseppe,  tear,  vernacular,  co,  vander, 
Nearest to to:  heretical,  dasyprocta,  will,  not,  can,  maximilian,  kapoor,  would, 
Nearest to american:  pulau,  british,  canadian,  homomorphism,  primigenius,  pair,  pontificia,  french, 
Nearest to years:  vma,  mediation,  pulau,  hennig,  abakan,  newsreel,  philanthropy,  xxviii, 
Nearest to time:  shabda,  mitral,  kapoor,  vdc,  agnostic,  mogul,  however,  operatorname, 
Nearest to their:  its,  his,  the,  her,  some,  posthumous,  pulau,  kapoor, 
Nearest to of:  ursus,  in,  vma,  original,  cebus,  archie,  including,  nine, 
Nearest to six:  seven,  eight,  five,  four,  three,  nine,  zero,  two, 
Nearest to been:  be,  become,  was,  were,  by,  ever,  had,  graphs, 
Nearest to not:  they,  still,  generally,  often,  to,  vert,  now,  you, 
Nearest to eight:  seven,  nine,  six,  five,  four,  three,  zero,  ursus, 
Nearest to also:  now,  still,  often,  which,  typically,  sometimes,  usually,  amazonas, 
Nearest to than:  or,  studd,  vdc,  but,  and,  much,  chronometers,  mastiff, 
Average loss at step  92000 :  4.67305505097
Average loss at step  94000 :  4.73028904641
Average loss at step  96000 :  4.69297025681
Average loss at step  98000 :  4.59498829353
Average loss at step  100000 :  4.68949806762
Nearest to d:  b,  cebus,  c,  rooks,  thibetanus,  UNK,  predicting,  otimes, 
Nearest to s:  his,  thibetanus,  pulau,  agouti,  thaler,  ursus,  circ,  and, 
Nearest to as:  unassigned,  agouti,  cebus,  microcebus,  ursus,  capuchin,  thibetanus,  iit, 
Nearest to states:  roshan,  antiprism,  trinomial,  giuseppe,  co,  tear,  vernacular,  circ, 
Nearest to to:  can,  dasyprocta,  would,  vert,  ursus,  will,  pulau,  heretical, 
Nearest to american:  british,  canadian,  pulau,  french,  primigenius,  homomorphism,  byzantine,  pontificia, 
Nearest to years:  vma,  mediation,  hennig,  newsreel,  pulau,  abakan,  decades,  three, 
Nearest to time:  shabda,  kapoor,  mitral,  vdc,  mogul,  point,  operatorname,  balaenoptera, 
Nearest to their:  its,  his,  her,  the,  some,  your,  posthumous,  pulau, 
Nearest to of:  ursus,  in,  vma,  original,  cebus,  agouti,  including,  and, 
Nearest to six:  seven,  eight,  five,  four,  three,  nine,  zero,  two, 
Nearest to been:  be,  become,  was,  ever,  by,  graphs,  had,  pacifist, 
Nearest to not:  still,  generally,  they,  often,  vert,  to,  now,  pulau, 
Nearest to eight:  seven,  six,  nine,  five,  four,  zero,  three,  ursus, 
Nearest to also:  now,  still,  often,  which,  typically,  sometimes,  usually,  amazonas, 
Nearest to than:  or,  studd,  vdc,  but,  chronometers,  much,  mastiff,  agouti,