Interview question

How do you handle large files efficiently in Python without running out of memory? Python में बड़ी files को बिना memory खत्म किए efficiently कैसे handle करें?

Answer

The key principle is to avoid loading the entire file into memory at once - process it in chunks or line by line using generators and iteration instead.

# BAD - loads the entire file into memory at once
with open('huge_file.txt') as f:
    lines = f.readlines()  # could use gigabytes of RAM for a large file
    for line in lines:
        process(line)

# GOOD - iterating the file object reads one line at a time
with open('huge_file.txt') as f:
    for line in f:  # memory-efficient, lazy line-by-line reading
        process(line.strip())

# Reading in fixed-size CHUNKS for binary or very long-line files
def read_in_chunks(file_path, chunk_size=1024 * 1024):  # 1MB chunks
    with open(file_path, 'rb') as f:
        while True:
            chunk = f.read(chunk_size)
            if not chunk:
                break
            yield chunk

for chunk in read_in_chunks('large_video.mp4'):
    process_chunk(chunk)

# Using a generator to process a large CSV without loading it all at once
import csv

def process_large_csv(filepath):
    with open(filepath) as f:
        reader = csv.DictReader(f)
        for row in reader:  # each row processed and discarded, not accumulated
            yield transform(row)

for result in process_large_csv('massive_data.csv'):
    save_to_database(result)

# For structured data too big for memory, consider pandas with chunksize
import pandas as pd
for chunk_df in pd.read_csv('huge.csv', chunksize=10000):
    process_dataframe_chunk(chunk_df)

मुख्य सिद्धांत यह है कि पूरी file को एक साथ memory में load न करें - इसे chunks या line by line generators और iteration से process करें।

# गलत - पूरी file एक साथ memory में
with open('huge_file.txt') as f:
    lines = f.readlines()  # बड़ी file के लिए gigabytes RAM लग सकता है
    for line in lines:
        process(line)

# सही - file object iterate करना, एक-एक line
with open('huge_file.txt') as f:
    for line in f:
        process(line.strip())

# Fixed-size CHUNKS में पढ़ना (binary या लंबी lines के लिए)
def read_in_chunks(file_path, chunk_size=1024 * 1024):  # 1MB
    with open(file_path, 'rb') as f:
        while True:
            chunk = f.read(chunk_size)
            if not chunk:
                break
            yield chunk

for chunk in read_in_chunks('large_video.mp4'):
    process_chunk(chunk)

# बड़ी CSV को generator से process करना
import csv

def process_large_csv(filepath):
    with open(filepath) as f:
        reader = csv.DictReader(f)
        for row in reader:
            yield transform(row)

for result in process_large_csv('massive_data.csv'):
    save_to_database(result)

# pandas में chunksize से बड़ी structured data
import pandas as pd
for chunk_df in pd.read_csv('huge.csv', chunksize=10000):
    process_dataframe_chunk(chunk_df)

Was this answer clear?

Subjects

Related questions