Parallelism with concurrent.futures

from concurrent.futures import ThreadPoolExecutor
import time

def download(url):
    # Simulate a network request that takes time
    time.sleep(1)
    return f"Downloaded {url}"

# Create a pool with 4 worker threads
# Think of it as hiring 4 assistants
executor = ThreadPoolExecutor(max_workers=4)

# Submit a task - returns immediately with a Future
# The task runs in the background
future = executor.submit(download, "https://example.com")
print("Task submitted - we can do other work here!")


...

from concurrent.futures import ThreadPoolExecutor

def fetch(url):
    # simulate work - in real code this would be an HTTP request
    return f"Content from {url}"

urls = ["url1", "url2", "url3", "url4"]

# The 'with' statement ensures automatic cleanup
# No need to call executor.shutdown() manually!
with ThreadPoolExecutor(max_workers=4) as executor:
    # map() is like the built-in map(), but parallel!
    # It automatically:
    # 1. Submits all tasks
    # 2. Collects results in order
    
...

from concurrent.futures import ProcessPoolExecutor

def heavy_compute(n):
    # CPU-intensive work: sum of squares
    # This keeps the CPU busy (no waiting)
    return sum(i * i for i in range(n))

numbers = [10_000_000, 10_000_000, 10_000_000, 10_000_000]

# ProcessPoolExecutor uses separate processes, not threads
# Each process gets its own Python interpreter and GIL
with ProcessPoolExecutor(max_workers=4) as executor:
    results = list(executor.map(heavy_compute, numbers))

print(results)
#
...

from concurrent.futures import ThreadPoolExecutor
import time

def task(n):
    # Simulate work that takes 'n' seconds
    time.sleep(n)
    return n * 2

executor = ThreadPoolExecutor()

# Submit a task that takes 2 seconds
future = executor.submit(task, 2)

# Check state immediately (task is running)
print(f"Done? {future.done()}")      # False
print(f"Running? {future.running()}")  # True

# Wait for result (blocks until complete)
result = future.result()
print(f"Result: {result}")  # 4

# No
...

from concurrent.futures import ThreadPoolExecutor

def failing_task():
    # This exception happens in a worker thread
    raise ValueError("Something went wrong")

executor = ThreadPoolExecutor()
future = executor.submit(failing_task)

# The exception is "stored" in the Future
# When we call .result(), it's re-raised here
try:
    future.result()
except ValueError as e:
    print(f"Caught: {e}")
# Output: Caught: Something went wrong

from concurrent.futures import ThreadPoolExecutor
import requests

def fetch_url(url):
    response = requests.get(url)
    return len(response.content)

urls = [
    "https://api.github.com/users/github",
    "https://api.github.com/users/microsoft",
    "https://api.github.com/users/google"
]

with ThreadPoolExecutor(max_workers=10) as executor:
    sizes = list(executor.map(fetch_url, urls))

print(f"Total bytes: {sum(sizes)}")

from concurrent.futures import ProcessPoolExecutor
import hashlib

def hash_password(password):
    return hashlib.pbkdf2_hmac('sha256', 
                                 password.encode(), 
                                 b'salt', 
                                 100000).hex()

passwords = ["pass123", "secret", "mypass"] * 1000

with ProcessPoolExecutor() as executor:
    hashed = list(executor.map(hash_password, passwords))

print(f"Hashed {len(hashed)} passwords")

import asyncio
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor

io_pool = ThreadPoolExecutor(20)
cpu_pool = ProcessPoolExecutor(8)

def read_file(path):
    with open(path) as f:
        return f.read()

def transform(text):
    return text.upper()

async def main(files):
    loop = asyncio.get_event_loop()

    # Stage 1: Read files (I/O bound)
    contents = await asyncio.gather(*[
        loop.run_in_executor(io_pool, read_file, f)
        for f in files
    ])

    # S
...

from multiprocessing import shared_memory
import numpy as np

# Create shared array
data = np.arange(10_000_000, dtype=np.int32)
shm = shared_memory.SharedMemory(create=True, size=data.nbytes)

# Write to shared block
shared_arr = np.ndarray(data.shape, dtype=data.dtype, buffer=shm.buf)
shared_arr[:] = data[:]

# Now multiple workers can read the same data:
# ✔ Zero copy
# ✔ Zero serialization
# ✔ Lightning-fast ML preprocessing

import asyncio
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor

async def master_pipeline(files):
    loop = asyncio.get_event_loop()
    io_pool = ThreadPoolExecutor(32)
    cpu_pool = ProcessPoolExecutor(8)

    # Stage 1 — Async orchestrates
    file_contents = await asyncio.gather(*[
        loop.run_in_executor(io_pool, open_file, f)
        for f in files
    ])

    # Stage 2 — CPU processing
    processed = await asyncio.gather(*[
        loop.run_in_executor(cpu_p
...