Как решал сложную задачу на проекте

import csv
from tqdm import tqdm

def process_large_csv(file_path):
    with open(file_path, 'r') as f:
        reader = csv.DictReader(f)
        for chunk in chunked(reader, 1000):  # Обрабатываем по 1000 строк
            process_chunk(chunk)

def chunked(iterable, size):
    chunk = []
    for item in tqdm(iterable, desc="Processing"):
        chunk.append(item)
        if len(chunk) == size:
            yield chunk
            chunk = []
    if chunk:
        yield chunk