🔵 Intermediate

Regular Expressions

📖 Lesson 24 ⏱ 40 min 🧪 5 questions 💻 3 exercises

🎯 Learning Objectives

Understand what regular expressions are and when to use them
Read and write common regex patterns and metacharacters
Use re.search, re.match, re.findall, re.sub, and re.split
Work with groups, named groups, and backreferences
Apply flags to control matching behaviour
Compile patterns for reuse with re.compile
Know when regex is the wrong tool

What are Regular Expressions?

A regular expression (regex) is a mini-language for describing text patterns. Instead of writing custom string-scanning loops, you express what you're looking for, and Python's re module does the searching.

import re

text = "Call us on 020-7946-0321 or 07700-900-123"

# Find all UK-style phone numbers — without regex, this is painful
phones = re.findall(r'\d[\d\-]{8,11}\d', text)
print(phones)   # ['020-7946-0321', '07700-900-123']

intro.py

Always use raw strings (r"...") for regex patterns. Without the r prefix, backslashes are interpreted by Python before the re module sees them — "\n" becomes a newline, not the two-character sequence backslash-n.

Pattern Syntax

Literal characters and metacharacters

import re

# Literals — match exactly
print(bool(re.search(r"cat", "the cat sat")))   # True
print(bool(re.search(r"dog", "the cat sat")))   # False

# . — any character except newline
print(bool(re.search(r"c.t", "cat")))   # True  (c-any-t)
print(bool(re.search(r"c.t", "cut")))   # True
print(bool(re.search(r"c.t", "ct")))    # False (no middle char)

# ^ and $ — start and end of string
print(bool(re.search(r"^hello", "hello world")))  # True
print(bool(re.search(r"^hello", "say hello")))    # False
print(bool(re.search(r"world$", "hello world")))  # True

literals_meta.py

Character classes

import re

# [abc] — one of the listed characters
print(re.findall(r"[aeiou]", "hello world"))  # ['e', 'o', 'o']

# [a-z] — character range
print(re.findall(r"[A-Z]", "Hello World"))    # ['H', 'W']

# [^abc] — NOT one of the listed characters
print(re.findall(r"[^aeiou\s]", "hello"))     # ['h', 'l', 'l']

# Shorthand classes
# \d = [0-9]          digit
# \D = [^0-9]         non-digit
# \w = [a-zA-Z0-9_]   word character
# \W = [^\w]          non-word character
# \s = [ \t\n\r\f\v]  whitespace
# \S = [^\s]          non-whitespace

print(re.findall(r"\d+", "abc 123 def 456"))  # ['123', '456']
print(re.findall(r"\w+", "hello, world!"))    # ['hello', 'world']

char_classes.py

Quantifiers

import re

# * — zero or more
print(re.findall(r"ab*", "a ab abb abbb"))   # ['a', 'ab', 'abb', 'abbb']

# + — one or more
print(re.findall(r"ab+", "a ab abb abbb"))   # ['ab', 'abb', 'abbb']

# ? — zero or one (optional)
print(re.findall(r"colou?r", "color colour"))  # ['color', 'colour']

# {n} — exactly n
print(re.findall(r"\d{4}", "2024 123 4567"))   # ['2024', '4567']

# {n,m} — between n and m (inclusive)
print(re.findall(r"\d{2,4}", "1 12 123 1234 12345"))  # ['12', '123', '1234', '1234']

# Greedy vs non-greedy (lazy)
html = "<b>bold</b> and <i>italic</i>"
print(re.findall(r"<.+>",  html))   # greedy  — matches as much as possible
print(re.findall(r"<.+?>", html))   # lazy    — matches as little as possible

quantifiers.py

Syntax	Meaning
`.`	Any character (except newline by default)
`^`	Start of string (or line with `MULTILINE`)
`$`	End of string (or line with `MULTILINE`)
`*`	0 or more (greedy)
`+`	1 or more (greedy)
`?`	0 or 1 (optional); also makes quantifier lazy
`{n,m}`	Between n and m repetitions
`[abc]`	Character class
`[^abc]`	Negated character class
`\d \w \s`	Digit / word char / whitespace
`\D \W \S`	Non-digit / non-word / non-whitespace
`\b`	Word boundary
`a\|b`	Alternation (a OR b)
`(...)`	Capturing group
`(?:...)`	Non-capturing group
`(?P<name>...)`	Named capturing group

Core `re` Functions

re.search() — find anywhere in string

import re

text = "The price is £42.50 today"

m = re.search(r'£(\d+\.\d{2})', text)
if m:
    print(m.group())    # £42.50  — full match
    print(m.group(1))   # 42.50   — first capture group
    print(m.start())    # 13      — start index
    print(m.end())      # 19      — end index
    print(m.span())     # (13, 19)

search.py

re.match() — only matches at start

import re

# match() only succeeds if the pattern is at the START of the string
print(re.match(r'\d+', '42 apples'))   # Match
print(re.match(r'\d+', 'I have 42'))   # None  — '4' not at start

# search() finds it anywhere
print(re.search(r'\d+', 'I have 42'))  # Match at position 7

match.py

re.fullmatch() — entire string must match

import re

# Validate that the WHOLE string is a valid email (simplified)
EMAIL = r'[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}'

print(bool(re.fullmatch(EMAIL, "alice@example.com")))  # True
print(bool(re.fullmatch(EMAIL, "not an email")))        # False
print(bool(re.fullmatch(EMAIL, "alice@example.com  "))) # False — trailing space

fullmatch.py

re.findall() and re.finditer()

import re

text = "Born: 1990-03-15, Graduated: 2012-06-01, Hired: 2015-09-20"

# findall — returns list of strings (or tuples if groups)
dates = re.findall(r'\d{4}-\d{2}-\d{2}', text)
print(dates)   # ['1990-03-15', '2012-06-01', '2015-09-20']

# With groups — returns list of tuples
parts = re.findall(r'(\d{4})-(\d{2})-(\d{2})', text)
print(parts)
# [('1990', '03', '15'), ('2012', '06', '01'), ('2015', '09', '20')]

# finditer — returns iterator of match objects (memory efficient)
for m in re.finditer(r'\d{4}-\d{2}-\d{2}', text):
    print(f"{m.group()} at position {m.start()}")

findall_finditer.py

re.sub() — search and replace

import re

# Simple replacement
result = re.sub(r'\s+', ' ', "too   many    spaces")
print(result)   # "too many spaces"

# Backreference in replacement: \1 refers to group 1
text = "2024-03-15"
reformatted = re.sub(r'(\d{4})-(\d{2})-(\d{2})', r'\3/\2/\1', text)
print(reformatted)   # 15/03/2024

# Replacement function — called for each match
def mask_card(m):
    digits = m.group()
    return '*' * (len(digits) - 4) + digits[-4:]

card_text = "Card: 1234567890123456"
print(re.sub(r'\d{12,16}', mask_card, card_text))
# Card: ************3456

# count= limits replacements
print(re.sub(r'a', 'X', "banana", count=2))   # bXnXna

sub.py

re.split()

import re

# Split on any whitespace or punctuation
tokens = re.split(r'[\s,;:]+', "one, two;  three: four")
print(tokens)   # ['one', 'two', 'three', 'four']

# Keep the delimiters using a capturing group
parts = re.split(r'(\s+)', "hello   world")
print(parts)    # ['hello', '   ', 'world']

split.py

Groups

import re

log = "2024-03-15 10:32:45 ERROR Connection refused"

# Capturing groups — numbered from 1, left to right
m = re.search(r'(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) (\w+) (.+)', log)
if m:
    print(m.group(0))   # entire match
    print(m.group(1))   # 2024-03-15
    print(m.group(2))   # 10:32:45
    print(m.group(3))   # ERROR
    print(m.group(4))   # Connection refused
    print(m.groups())   # ('2024-03-15', '10:32:45', 'ERROR', 'Connection refused')

groups.py

Named groups

import re

# Named groups: (?P<name>pattern)
LOG_PATTERN = re.compile(
    r'(?P<date>\d{4}-\d{2}-\d{2}) '
    r'(?P<time>\d{2}:\d{2}:\d{2}) '
    r'(?P<level>\w+) '
    r'(?P<message>.+)'
)

log = "2024-03-15 10:32:45 ERROR Connection refused"
m = LOG_PATTERN.search(log)
if m:
    print(m.group("date"))     # 2024-03-15
    print(m.group("level"))    # ERROR
    print(m.groupdict())
    # {'date': '2024-03-15', 'time': '10:32:45',
    #  'level': 'ERROR', 'message': 'Connection refused'}

named_groups.py

Non-capturing groups

import re

# (?:...) — group for structure/alternation but don't capture
# Matches "colour" or "color", captures only the full word
m = re.search(r'(?:colour|color)', "The colour is red")
print(m.group())   # colour

# Useful in alternation without polluting group numbers
dates = re.findall(r'\b(?:Jan|Feb|Mar|Apr|May|Jun) \d{1,2}\b',
                   "Jan 5, Feb 14, and Dec 25")
print(dates)   # ['Jan 5', 'Feb 14']

non_capturing.py

Flags

import re

text = "Hello World\nhello again"

# re.IGNORECASE (re.I) — case-insensitive matching
print(re.findall(r'hello', text, re.IGNORECASE))
# ['Hello', 'hello']

# re.MULTILINE (re.M) — ^ and $ match start/end of each LINE
print(re.findall(r'^\w+', text, re.MULTILINE))
# ['Hello', 'hello']

# re.DOTALL (re.S) — . matches newlines too
m = re.search(r'Hello.+again', text, re.DOTALL)
print(bool(m))   # True (. crossed the newline)

# re.VERBOSE (re.X) — allow whitespace and comments in pattern
EMAIL = re.compile(r'''
    [a-zA-Z0-9._%+\-]+  # local part
    @                    # at sign
    [a-zA-Z0-9.\-]+      # domain
    \.                   # dot
    [a-zA-Z]{2,}         # TLD
''', re.VERBOSE)

print(bool(EMAIL.match("alice@example.com")))  # True

# Combine flags with |
print(re.findall(r'^hello', text, re.IGNORECASE | re.MULTILINE))
# ['Hello', 'hello']

flags.py

Compiling Patterns

If you use the same pattern many times, compile it once with re.compile() to avoid repeated parsing overhead and to keep the code clean:

import re

# Compile once — use the pattern object's methods
DATE_RE = re.compile(r'\b(\d{4})-(\d{2})-(\d{2})\b')

texts = [
    "Invoice dated 2024-03-15",
    "Shipped 2024-04-01, delivered 2024-04-03",
    "No dates here",
]

for text in texts:
    matches = DATE_RE.findall(text)
    if matches:
        for year, month, day in matches:
            print(f"{day}/{month}/{year}")

# Compiled patterns have all the same methods:
# DATE_RE.search(), .match(), .fullmatch()
# DATE_RE.findall(), .finditer()
# DATE_RE.sub(), .split()

compile.py

For patterns used more than once in a module, compile them at module level (outside any function). This is both a performance optimisation and a good way to name and document your patterns.

Lookahead & Lookbehind

These zero-width assertions match a position without consuming characters:

import re

prices = "£10, $20, €30, 40 USD"

# Positive lookahead (?=...) — match word followed by something
print(re.findall(r'\d+(?= USD)', prices))   # ['40']

# Negative lookahead (?!...) — match word NOT followed by something
print(re.findall(r'\d+(?! USD)', prices))   # ['10', '20', '30']

# Positive lookbehind (?<=...) — match word preceded by something
print(re.findall(r'(?<=£)\d+', prices))    # ['10']

# Negative lookbehind (?<!...) — match word NOT preceded by something
print(re.findall(r'(?<!\$)\d+', prices))   # ['10', '30', '40']

# Practical: extract numbers after a currency symbol
amounts = re.findall(r'(?<=[£$€])\d+', prices)
print(amounts)   # ['10', '20', '30']

lookahead.py

Common Regex Patterns

import re

# ── Email (simplified) ──
EMAIL    = re.compile(r'[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}')

# ── URL (simplified) ──
URL      = re.compile(r'https?://[^\s"\'<>]+')

# ── UK postcode ──
POSTCODE = re.compile(r'\b[A-Z]{1,2}\d[A-Z\d]? ?\d[A-Z]{2}\b', re.IGNORECASE)

# ── ISO 8601 date ──
ISO_DATE = re.compile(r'\b\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])\b')

# ── IPv4 address ──
IPV4     = re.compile(r'\b(?:\d{1,3}\.){3}\d{1,3}\b')

# ── Hex colour ──
HEX_COL  = re.compile(r'#(?:[0-9a-fA-F]{3}){1,2}\b')

test_cases = [
    ("Email",    EMAIL,    "Contact alice@example.com or bob@test.co.uk"),
    ("URL",      URL,      "Visit https://python.org or http://docs.python.org/3/"),
    ("Postcode", POSTCODE, "Send to EC1A 1BB or SW1A 0AA"),
    ("Date",     ISO_DATE, "From 2024-01-01 to 2024-12-31"),
    ("IPv4",     IPV4,     "Servers: 192.168.1.1 and 10.0.0.255"),
    ("Hex",      HEX_COL,  "Colours: #fff #3a86ff #FF5733"),
]

for label, pattern, text in test_cases:
    print(f"{label:10}: {pattern.findall(text)}")

common_patterns.py

When NOT to Use Regex

Parsing HTML/XML — use BeautifulSoup or lxml. Regex cannot reliably parse nested structures.
Parsing JSON — use json.loads().
Simple string operations — str.startswith(), str.endswith(), str.replace(), str.split() are faster and more readable.
Complex grammars (programming languages, config files) — use a proper parser library.

text = "hello world"

# ❌ Regex overkill for simple prefix check
if re.match(r'^hello', text):
    print("starts with hello")

# ✓ Plain string method — clearer and faster
if text.startswith("hello"):
    print("starts with hello")

# ❌ Regex to check extension
if re.search(r'\.py$', filename):
    pass

# ✓ pathlib
from pathlib import Path
if Path(filename).suffix == ".py":
    pass

when_not_to.py

Primary sources: Python Docs — re module · Python Docs — Regular Expression HOWTO · regex101.com — interactive regex tester

🤖

Ask your AI tutor! Stuck on a tricky pattern? Paste your test string and describe what you want to match — building regex interactively with explanations is one of the best ways to learn. Also great: ask for a verbose (re.VERBOSE) version of any pattern with comments.

💻 Exercises

01 Log Parser

Write a function parse_logs(log_text) that extracts structured data from Apache-style log lines:

log_text = """
192.168.1.1 - - [15/Mar/2024:10:32:45 +0000] "GET /index.html HTTP/1.1" 200 1024
10.0.0.5 - alice [15/Mar/2024:10:33:01 +0000] "POST /api/data HTTP/2" 201 512
172.16.0.3 - - [15/Mar/2024:10:33:22 +0000] "GET /missing HTTP/1.1" 404 0
"""

Return a list of dicts with keys: ip, user, timestamp, method, path, status (int), size (int).

Show solution

import re

LOG_RE = re.compile(
    r'(?P<ip>\S+) \S+ (?P<user>\S+) '
    r'\[(?P<timestamp>[^\]]+)\] '
    r'"(?P<method>\w+) (?P<path>\S+) [^"]*" '
    r'(?P<status>\d{3}) (?P<size>\d+)'
)

def parse_logs(log_text):
    records = []
    for m in LOG_RE.finditer(log_text):
        d = m.groupdict()
        d["status"] = int(d["status"])
        d["size"]   = int(d["size"])
        if d["user"] == "-":
            d["user"] = None
        records.append(d)
    return records

log_text = """
192.168.1.1 - - [15/Mar/2024:10:32:45 +0000] "GET /index.html HTTP/1.1" 200 1024
10.0.0.5 - alice [15/Mar/2024:10:33:01 +0000] "POST /api/data HTTP/2" 201 512
172.16.0.3 - - [15/Mar/2024:10:33:22 +0000] "GET /missing HTTP/1.1" 404 0
"""

for entry in parse_logs(log_text):
    print(entry)

02 Text Cleaner

Write a function clean_text(text) that uses re.sub() to:

Collapse multiple whitespace characters (spaces, tabs, newlines) into a single space
Remove all HTML tags (anything of the form <…>)
Normalise smart/curly quotes (" " ' ') to straight quotes (" and ')
Strip leading and trailing whitespace from the result

Show solution

import re

def clean_text(text):
    # 1. Remove HTML tags
    text = re.sub(r'<[^>]+>', '', text)
    # 2. Normalise curly quotes
    text = re.sub(r'[\u201c\u201d]', '"', text)
    text = re.sub(r'[\u2018\u2019]', "'", text)
    # 3. Collapse whitespace
    text = re.sub(r'\s+', ' ', text)
    # 4. Strip
    return text.strip()

sample = """<h1>Hello  World</h1>
<p>She said \u201cHello\u201d and he replied \u2018Hi\u2019.</p>
   Extra   spaces   here.  """

print(clean_text(sample))
# Hello World She said "Hello" and he replied 'Hi'. Extra spaces here.

03 Data Extractor

Write a function extract_contacts(text) that scans a block of text and returns a dict with:

"emails" — list of all email addresses found
"phones" — list of all UK phone numbers (07xxx xxxxxx or 01xxx xxxxxx patterns)
"urls" — list of all http/https URLs found

Show solution

import re

EMAIL_RE = re.compile(r'[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}')
PHONE_RE = re.compile(r'\b0[17]\d{3}[\s\-]?\d{6}\b')
URL_RE   = re.compile(r'https?://[^\s"\'<>,]+')

def extract_contacts(text):
    return {
        "emails": EMAIL_RE.findall(text),
        "phones": PHONE_RE.findall(text),
        "urls":   URL_RE.findall(text),
    }

sample = """
Please contact sales@example.com or support@help.co.uk for assistance.
Call us on 07700 123456 or 01234-567890.
Visit https://www.example.com or see the docs at https://docs.example.com/api.
"""

result = extract_contacts(sample)
for key, values in result.items():
    print(f"{key}: {values}")

Regular Expressions

🎯 Learning Objectives

What are Regular Expressions?

Pattern Syntax

Literal characters and metacharacters

Character classes

Quantifiers

Core re Functions

re.search() — find anywhere in string

re.match() — only matches at start

re.fullmatch() — entire string must match

re.findall() and re.finditer()

re.sub() — search and replace

re.split()

Groups

Named groups

Non-capturing groups

Flags

Compiling Patterns

Lookahead & Lookbehind

Common Regex Patterns

When NOT to Use Regex

💻 Exercises

Core `re` Functions