Python foundations

# Check what hardware you actually got.
import torch
print(torch.__version__, torch.cuda.is_available())
if torch.cuda.is_available():
    print(torch.cuda.get_device_name(0))

!nvidia-smi        # ! runs a shell command

# Install what is missing. -q keeps the output short.
!pip install -q transformers datasets tiktoken

# Persist anything you care about — the runtime is deleted when it disconnects.
from google.colab import drive
drive.mount("/content/drive")
torch.save(model.state_dict(), "/content/drive/MyDrive/model.pt")

# Timing and inspection:
%time  slow_function()
%%time
# times the whole cell

?torch.softmax          # docstring
help(torch.softmax)     # same, printed
torch.softmax??         # source, where available

print(7 / 2)        # 3.5  — true division, ALWAYS a float
print(7 // 2)       # 3    — floor division
print(-7 // 2)      # -4   — floors toward -infinity, not toward zero
print(-7 % 2)       # 1    — sign follows the divisor, unlike C
print(2 ** 10)      # 1024 — exponentiation

print(0.1 + 0.2 == 0.3)                  # False — binary floating point
print(abs(0.1 + 0.2 - 0.3) < 1e-9)       # the correct comparison

import math
print(math.isclose(0.1 + 0.2, 0.3))      # or use this

print(True + True)                       # 2 — bool subclasses int
print(sum([True, False, True]))          # 2 — which is how you count matches

a = [0, 1, 2, 3, 4, 5]

print(a[2])        # 2
print(a[-1])       # 5
print(a[1:4])      # [1, 2, 3]      — stop excluded
print(a[:3])       # [0, 1, 2]
print(a[3:])       # [3, 4, 5]
print(a[::2])      # [0, 2, 4]
print(a[::-1])     # [5, 4, 3, 2, 1, 0]

print(a[10])       # IndexError
print(a[3:100])    # [3, 4, 5] — no error, which is why slice bugs hide

# Slicing a LIST copies (shallowly). Slicing a NUMPY ARRAY is a view.
import numpy as np
x = np.arange(6)
v = x[1:4]
v[0] = 99
print(x)           # [ 0 99  2  3  4  5]  — the original changed!
safe = x[1:4].copy()   # take an explicit copy when you mean one

# Strings slice the same way but are immutable.
s = "tokenization"
print(s[:5], s[-4:], s[::-1])

vocab = {"the": 0, "cat": 1, "sat": 2}

print(vocab["cat"])            # 1
print(vocab.get("dog", -1))    # -1 — no KeyError
vocab["dog"] = 3               # insert or overwrite

for word, idx in vocab.items():
    print(word, idx)

# Inverting a dict — needed constantly for id -> token.
itos = {i: s for s, i in vocab.items()}

# Counting: Counter, or setdefault, or defaultdict.
from collections import Counter, defaultdict
counts = Counter("mississippi")
print(counts.most_common(3))          # [('i', 4), ('s', 4), ('p', 2)]

groups = defaultdict(list)
for word in ["cat", "cot", "dog"]:
    groups[word[0]].append(word)      # no need to initialise the list

# Keys must be hashable, i.e. immutable.
d = {(1, 2): "tuple key works"}
# d = {[1, 2]: "fails"}   -> TypeError: unhashable type: 'list'

import copy

a = [1, 2, 3]
b = a                 # same object
b.append(4)
print(a)              # [1, 2, 3, 4]  — a changed
print(a is b)         # True — identity, not equality

c = a[:]              # shallow copy
c.append(5)
print(a)              # unchanged

nested = [[1, 2], [3, 4]]
shallow = nested[:]           # outer copied, INNER LISTS SHARED
shallow[0].append(99)
print(nested)                 # [[1, 2, 99], [3, 4]] — still changed

deep = copy.deepcopy(nested)  # fully independent

# The classic:
grid = [[0] * 3] * 2          # both rows are THE SAME list
grid[0][0] = 9
print(grid)                   # [[9, 0, 0], [9, 0, 0]]

grid = [[0] * 3 for _ in range(2)]   # correct

# Same trap in function defaults — evaluated ONCE at definition:
def bad(x, acc=[]):
    acc.append(x)
    return acc
print(bad(1), bad(2))         # [1, 2] [1, 2]  — shared across calls

def good(x, acc=None):
    acc = [] if acc is None else acc
    acc.append(x)
    return acc

def tokenize(text, lowercase=True, max_len=None):
    """Split text into tokens.

    Args:
        text: the input string.
        lowercase: fold case before splitting.
        max_len: truncate to this many tokens; None keeps all.
    Returns:
        A list of tokens.
    """
    if lowercase:
        text = text.lower()
    tokens = text.split()
    return tokens[:max_len] if max_len else tokens

tokenize("Hello World")                  # positional
tokenize("Hello World", max_len=1)       # keyword — always clearer

# Multiple returns are a tuple, and unpack directly.
def stats(xs):
    return min(xs), max(xs), sum(xs) / len(xs)

lo, hi, mean = stats([1, 2, 3, 4])

# *args and **kwargs collect the rest.
def log(prefix, *values, sep=" ", **meta):
    print(prefix, sep.join(map(str, values)), meta)

# Type hints: not enforced at runtime, but they make editors useful and
# document the contract. Worth writing in anything you will reread.
def cosine(a: "np.ndarray", b: "np.ndarray") -> float:
    import numpy as np
    return float(a @ b / (np.linalg.norm(a) * np.linalg.norm(b)))

# for over an iterable — you rarely need range(len(x))
for token in tokens:
    process(token)

# enumerate when you need the index too
for i, token in enumerate(tokens):
    print(i, token)

for i, token in enumerate(tokens, start=1):     # 1-based
    print(i, token)

# zip to walk several sequences together — stops at the SHORTEST
for token, score in zip(tokens, scores):
    print(token, score)

for token, score in zip(tokens, scores, strict=True):   # 3.10+: error if unequal
    pass

# Comprehensions: build a new list from an old one. Faster and clearer than
# append in a loop, and the standard idiom in this kind of code.
lengths = [len(t) for t in tokens]
long = [t for t in tokens if len(t) > 3]
upper = [t.upper() if t.isalpha() else t for t in tokens]   # note: if/else BEFORE for

pairs = [(i, j) for i in range(3) for j in range(3) if i != j]   # nested

squares = {t: len(t) for t in tokens}        # dict comprehension
unique = {t.lower() for t in tokens}         # set comprehension
gen = (len(t) for t in tokens)               # generator — lazy, no list built

# Initialise before accumulating, or you get NameError on the first iteration.
total, best = 0, float("-inf")
for score in scores:
    total += score
    best = max(best, score)

import numpy as np

x = np.array([[1., 2., 3.], [4., 5., 6.]])
print(x.shape, x.dtype, x.ndim)          # (2, 3) float64 2

np.zeros((2, 3)); np.ones(4); np.eye(3)
np.arange(0, 1, 0.25)                    # [0, 0.25, 0.5, 0.75]
np.linspace(0, 1, 5)                     # [0, 0.25, 0.5, 0.75, 1] — includes 1

# Vectorise: operate on whole arrays, never element by element in Python.
print(x * 2)
print(x @ x.T)                           # matrix multiply
print(x.sum(axis=0))                     # down the columns -> shape (3,)
print(x.mean(axis=1, keepdims=True))     # across rows, shape kept as (2, 1)

# keepdims is what makes broadcasting work for normalisation:
print(x / np.linalg.norm(x, axis=1, keepdims=True))

# Boolean masking
print(x[x > 3])                          # 1-D array of the matches
print(np.where(x > 3, x, 0))             # elementwise conditional

# Reproducible randomness — the modern API, one generator you pass around.
rng = np.random.default_rng(seed=42)
rng.standard_normal((2, 3))
rng.integers(0, 10, size=5)
rng.choice(["a", "b", "c"], size=3, replace=False)
rng.permutation(10)

# Reshaping: -1 means 'infer this dimension'.
print(x.reshape(3, 2), x.reshape(-1), x.T.shape)