import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'       # suppress TF hardware warnings
os.environ['TF_ENABLE_ONEDNN_OPTS'] = '0'       # suppress oneDNN floating-point notes

import tensorflow as tf
import tensorflow_hub as hub
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import (
    classification_report, accuracy_score,
    confusion_matrix, ConfusionMatrixDisplay
)

print(f"TensorFlow version: {tf.__version__}")
print(f"TensorFlow Hub version: {hub.__version__}")
print("All imports loaded.")

WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
I0000 00:00:1779299471.032155     707 cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.

WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
I0000 00:00:1779299472.431723     707 cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.

TensorFlow version: 2.21.0
TensorFlow Hub version: 0.16.1
All imports loaded.

df = pd.read_csv('hotel_reviews.csv')

print(f"Shape: {df.shape}")
print(f"\nClass distribution:")
print(df['Is_Response'].value_counts())

# Label encode — happy=0, not happy=1
le = LabelEncoder()
df['label'] = le.fit_transform(df['Is_Response'])
print(f"\nLabel encoding: {dict(zip(le.classes_, le.transform(le.classes_)))}")
df.head(3)

Shape: (1000, 3)

Class distribution:
Is_Response
happy        543
not happy    457
Name: count, dtype: int64

Label encoding: {'happy': np.int64(0), 'not happy': np.int64(1)}

X = df['Description'].astype(str)
y = df['label']

# 70% train, 30% temp
X_train, X_temp, y_train, y_temp = train_test_split(
    X, y, test_size=0.30, random_state=42, stratify=y
)
# 15% val, 15% test from the 30% temp
X_val, X_test, y_val, y_test = train_test_split(
    X_temp, y_temp, test_size=0.50, random_state=42, stratify=y_temp
)

print(f"Train:      {len(X_train):>4} rows ({len(X_train)/len(df)*100:.0f}%)")
print(f"Validation: {len(X_val):>4} rows ({len(X_val)/len(df)*100:.0f}%)")
print(f"Test:       {len(X_test):>4} rows ({len(X_test)/len(df)*100:.0f}%)")

Train:       700 rows (70%)
Validation:  150 rows (15%)
Test:        150 rows (15%)

USE_URL = "https://tfhub.dev/google/universal-sentence-encoder/4"

print("Loading Universal Sentence Encoder from TensorFlow Hub...")
print("(First run downloads ~1GB and caches locally — subsequent runs are faster)")
embed = hub.load(USE_URL)
print("USE loaded successfully.")

# Quick sanity check
test_vecs = embed(["The hotel was great.", "Terrible experience."])
print(f"\nEmbedding shape per sentence: {test_vecs.shape}")  # should be (2, 512)

Loading Universal Sentence Encoder from TensorFlow Hub...
(First run downloads ~1GB and caches locally — subsequent runs are faster)

USE loaded successfully.

Embedding shape per sentence: (2, 512)

import time

def encode_texts(texts, batch_size=64):
    """
    Encodes a list of strings into 512-dim USE vectors, processed in batches.
    Batching avoids memory issues on large inputs.
    Returns a single numpy array of shape (n_samples, 512).
    """
    embeddings = []
    texts_list = list(texts)
    for i in range(0, len(texts_list), batch_size):
        batch = texts_list[i:i + batch_size]
        embeddings.append(embed(batch).numpy())
    return np.vstack(embeddings)

print("Encoding train, validation, and test sets...")
t0 = time.time()

X_train_emb = encode_texts(X_train)
X_val_emb   = encode_texts(X_val)
X_test_emb  = encode_texts(X_test)

elapsed = time.time() - t0
print(f"Encoding complete in {elapsed:.1f}s")
print(f"Train embedding shape:  {X_train_emb.shape}")
print(f"Val embedding shape:    {X_val_emb.shape}")
print(f"Test embedding shape:   {X_test_emb.shape}")

Encoding train, validation, and test sets...
Encoding complete in 0.1s
Train embedding shape:  (700, 512)
Val embedding shape:    (150, 512)
Test embedding shape:   (150, 512)

tf.random.set_seed(42)

model = tf.keras.Sequential([
    # Input: 512-dim USE embedding
    tf.keras.layers.InputLayer(shape=(512,)),

    # Hidden layer 1: 256 units, ReLU activation
    # ReLU: f(x) = max(0, x) — introduces non-linearity so the model can learn
    # patterns that a simple linear combination of inputs can't capture
    tf.keras.layers.Dense(256, activation='relu'),
    tf.keras.layers.Dropout(0.3),   # drop 30% of neurons per step during training

    # Hidden layer 2: 128 units, ReLU
    tf.keras.layers.Dense(128, activation='relu'),
    tf.keras.layers.Dropout(0.3),

    # Output: single sigmoid unit — probability of 'not happy' (label=1)
    tf.keras.layers.Dense(1, activation='sigmoid')
])

model.compile(
    optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),
    loss='binary_crossentropy',
    metrics=['accuracy']
)

model.summary()

Model: "sequential"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓
┃ Layer (type)                    ┃ Output Shape           ┃       Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩
│ dense (Dense)                   │ (None, 256)            │       131,328 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dropout (Dropout)               │ (None, 256)            │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dense_1 (Dense)                 │ (None, 128)            │        32,896 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dropout_1 (Dropout)             │ (None, 128)            │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dense_2 (Dense)                 │ (None, 1)              │           129 │
└─────────────────────────────────┴────────────────────────┴───────────────┘

 Total params: 164,353 (642.00 KB)

 Trainable params: 164,353 (642.00 KB)

 Non-trainable params: 0 (0.00 B)

TOTAL_STEPS = 100
STEP_SIZE   = 10
BATCH_SIZE  = 32
EPOCHS      = TOTAL_STEPS // STEP_SIZE   # = 10

print(f"TOTAL_STEPS={TOTAL_STEPS}, STEP_SIZE={STEP_SIZE}")
print(f"Epochs: {EPOCHS}, Batch size: {BATCH_SIZE}")
print("\nTraining...")

t0 = time.time()
history = model.fit(
    X_train_emb, y_train.values,
    validation_data=(X_val_emb, y_val.values),
    epochs=EPOCHS,
    batch_size=BATCH_SIZE,
    verbose=1
)

print(f"\nTraining complete in {time.time()-t0:.1f} seconds.")

TOTAL_STEPS=100, STEP_SIZE=10
Epochs: 10, Batch size: 32

Training...

Epoch 1/10

 1/22 ━━━━━━━━━━━━━━━━━━━━ 17s 828ms/step - accuracy: 0.4688 - loss: 0.6956


22/22 ━━━━━━━━━━━━━━━━━━━━ 1s 10ms/step - accuracy: 0.9286 - loss: 0.4400 - val_accuracy: 1.0000 - val_loss: 0.1033

Epoch 2/10

 1/22 ━━━━━━━━━━━━━━━━━━━━ 1s 94ms/step - accuracy: 1.0000 - loss: 0.1154


22/22 ━━━━━━━━━━━━━━━━━━━━ 0s 5ms/step - accuracy: 1.0000 - loss: 0.0330 - val_accuracy: 1.0000 - val_loss: 0.0021

Epoch 3/10

 1/22 ━━━━━━━━━━━━━━━━━━━━ 0s 46ms/step - accuracy: 1.0000 - loss: 0.0040


22/22 ━━━━━━━━━━━━━━━━━━━━ 0s 4ms/step - accuracy: 1.0000 - loss: 0.0018 - val_accuracy: 1.0000 - val_loss: 5.3678e-04

fig, axes = plt.subplots(1, 2, figsize=(12, 4))

# Loss curves
axes[0].plot(history.history['loss'],     label='Train Loss',      color='#457B9D', linewidth=2)
axes[0].plot(history.history['val_loss'], label='Validation Loss', color='#E63946', linewidth=2, linestyle='--')
axes[0].set_title('Loss per Epoch')
axes[0].set_xlabel('Epoch')
axes[0].set_ylabel('Binary Crossentropy Loss')
axes[0].legend()
axes[0].set_xticks(range(EPOCHS))
axes[0].set_xticklabels(range(1, EPOCHS+1))

# Accuracy curves
axes[1].plot(history.history['accuracy'],     label='Train Accuracy',      color='#2A9D8F', linewidth=2)
axes[1].plot(history.history['val_accuracy'], label='Validation Accuracy', color='#E9C46A', linewidth=2, linestyle='--')
axes[1].set_title('Accuracy per Epoch')
axes[1].set_xlabel('Epoch')
axes[1].set_ylabel('Accuracy')
axes[1].set_ylim([0.5, 1.05])
axes[1].legend()
axes[1].set_xticks(range(EPOCHS))
axes[1].set_xticklabels(range(1, EPOCHS+1))

plt.suptitle('USE Deep Learning Model — Training History', fontsize=13, fontweight='bold')
plt.tight_layout()
plt.show()

# Epoch-by-epoch table
print(f"\n{'Epoch':>6} {'Train Loss':>12} {'Val Loss':>10} {'Train Acc':>10} {'Val Acc':>10}")
print("-" * 55)
for i in range(EPOCHS):
    print(f"{i+1:>6} {history.history['loss'][i]:>12.4f} {history.history['val_loss'][i]:>10.4f} "
          f"{history.history['accuracy'][i]:>10.4f} {history.history['val_accuracy'][i]:>10.4f}")

 Epoch   Train Loss   Val Loss  Train Acc    Val Acc
-------------------------------------------------------
     1       0.4400     0.1033     0.9286     1.0000
     2       0.0330     0.0021     1.0000     1.0000
     3       0.0018     0.0005     1.0000     1.0000
     4       0.0008     0.0003     1.0000     1.0000
     5       0.0006     0.0003     1.0000     1.0000
     6       0.0004     0.0002     1.0000     1.0000
     7       0.0003     0.0002     1.0000     1.0000
     8       0.0002     0.0001     1.0000     1.0000
     9       0.0003     0.0001     1.0000     1.0000
    10       0.0002     0.0001     1.0000     1.0000

# Validation set evaluation
val_loss, val_acc = model.evaluate(X_val_emb, y_val.values, verbose=0)
print(f"Validation Loss:     {val_loss:.6f}")
print(f"Validation Accuracy: {val_acc:.4f}")

print()

# Test set evaluation — final holdout
test_loss, test_acc = model.evaluate(X_test_emb, y_test.values, verbose=0)
print(f"Test Loss:           {test_loss:.6f}")
print(f"Test Accuracy:       {test_acc:.4f}")

Validation Loss:     0.000093
Validation Accuracy: 1.0000

Test Loss:           0.000084
Test Accuracy:       1.0000

# Generate predictions from sigmoid probabilities
# Threshold at 0.5: above = not happy (1), below = happy (0)
y_pred_prob = model.predict(X_test_emb, verbose=0)
y_pred      = (y_pred_prob > 0.5).astype(int).flatten()

print("Classification Report — Test Set")
print("=" * 55)
print(classification_report(y_test, y_pred, target_names=le.classes_))

# Sample predicted probabilities
print("\nSample Predicted Probabilities (first 8 test reviews):")
print(f"{'Review (truncated)':<55} {'P(not happy)':>13} {'Predicted':>10} {'Actual':>10}")
print("-" * 95)
for text, prob, pred, true in zip(
    list(X_test)[:8],
    y_pred_prob.flatten()[:8],
    y_pred[:8],
    y_test.values[:8]
):
    print(f"{str(text)[:54]:<55} {prob:>13.4f} {le.classes_[pred]:>10} {le.classes_[true]:>10}")

Classification Report — Test Set
=======================================================
              precision    recall  f1-score   support

       happy       1.00      1.00      1.00        82
   not happy       1.00      1.00      1.00        68

    accuracy                           1.00       150
   macro avg       1.00      1.00      1.00       150
weighted avg       1.00      1.00      1.00       150

Sample Predicted Probabilities (first 8 test reviews):
Review (truncated)                                       P(not happy)  Predicted     Actual
-----------------------------------------------------------------------------------------------
Loved every minute of our stay. The spa facilities wer         0.0004      happy      happy
Breakfast was substandard and the coffee was undrinkab         0.9999  not happy  not happy
Overpriced for what you get. Much better options avail         0.9999  not happy  not happy
Our room had not been properly cleaned when we arrived         1.0000  not happy  not happy
Front desk staff were unhelpful and dismissive when we         1.0000  not happy  not happy
No hot water in the morning despite multiple complaint         1.0000  not happy  not happy
The hotel did not match the photos online. Rooms were          0.9999  not happy  not happy
Bed was uncomfortable and the pillows were flat. Worst         1.0000  not happy  not happy

# Confusion matrix
cm = confusion_matrix(y_test, y_pred)
disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=le.classes_)
fig, ax = plt.subplots(figsize=(5, 4))
disp.plot(ax=ax, colorbar=False, cmap='Blues')
ax.set_title('USE Deep Learning Model — Test Set Confusion Matrix')
plt.tight_layout()
plt.show()

print(f"\nTrue Positives  (happy correctly classified):      {cm[0,0]}")
print(f"True Negatives  (not happy correctly classified): {cm[1,1]}")
print(f"False Positives (happy predicted as not happy):   {cm[0,1]}")
print(f"False Negatives (not happy predicted as happy):   {cm[1,0]}")

True Positives  (happy correctly classified):      82
True Negatives  (not happy correctly classified): 68
False Positives (happy predicted as not happy):   0
False Negatives (not happy predicted as happy):   0

# Summary table
print("Model Performance Summary")
print("=" * 45)
print(f"{'Metric':<30} {'Value'}")
print("-" * 45)
print(f"{'Test Accuracy':<30} {test_acc:.4f}")
print(f"{'Test Loss':<30} {test_loss:.6f}")
print(f"{'Precision (happy)':<30} 1.00")
print(f"{'Recall (happy)':<30} 1.00")
print(f"{'Precision (not happy)':<30} 1.00")
print(f"{'Recall (not happy)':<30} 1.00")
print(f"{'F1 Score (weighted avg)':<30} 1.00")
print(f"{'Embedding dimensions':<30} 512")
print(f"{'Total trainable params':<30} 164,353")
print(f"{'Training epochs':<30} {EPOCHS}")
print(f"{'Batch size':<30} {BATCH_SIZE}")
print("=" * 45)

Model Performance Summary
=============================================
Metric                         Value
---------------------------------------------
Test Accuracy                  1.0000
Test Loss                      0.000084
Precision (happy)              1.00
Recall (happy)                 1.00
Precision (not happy)          1.00
Recall (not happy)             1.00
F1 Score (weighted avg)        1.00
Embedding dimensions           512
Total trainable params         164,353
Training epochs                10
Batch size                     32
=============================================

	User_ID	Description	Is_Response	label
0	USER_0001	The hotel was absolutely wonderful. Staff were...	happy	0
1	USER_0002	Waited over an hour for room service and the f...	not happy	1
2	USER_0003	Great wifi and comfortable work desk made this...	happy	0

Dimension	TF-IDF + Logistic Regression	USE + Dense Network
Feature source	Word frequency counts (sparse)	Pre-trained sentence embeddings (dense 512-dim)
Text cleaning	Required (stopwords, lemmatization)	None — USE handles raw text
Training data needed	Labeled examples only	Pre-trained + labeled examples
Vocabulary	Fixed to training corpus	Universal — handles unseen words
Handles paraphrase?	Rarely — word overlap dependent	Yes — semantic space captures meaning
Test accuracy	100%	100%

Week 10 — Deep Learning Sentiment Analysis: Universal Sentence Encoder¶

Overview¶

TF 2.0 Compatibility¶

TOTAL_STEPS and STEP_SIZE¶

Data Split¶

Step 1: Imports¶

Step 2: Load Data and Label Encode¶

Step 3: Three-Way Train / Validation / Test Split¶

Step 4: Load the Universal Sentence Encoder¶

Step 5: Build the Keras Classification Model¶

Step 6: Train the Model¶

Step 7: Training History — Loss and Accuracy Curves¶

Reading the Training Curves¶

Step 8: Model Evaluation¶

Step 9: Week 9 vs. Week 10 — Model Comparison¶

Observations and Interpretation¶

References¶