import psutil
import pandas as pd
import csv
svmem = psutil.virtual_memory()
print (svmem.available)
PATH = r”C:\ tmp \dataset\tf_train.csv”
df_sample = pd.read_csv(PATH, nrows=10)
df_sample_size = df_sample.memory_usage(index=True).sum()
print (df_sample_size)
print (df_sample)
# define a chunksize that would occupy a maximum of 1Gb
# we divide by 10 because we have selected 10 lines in our df_sample
my_chunk = (1000000000 / df_sample_size)/10
my_chunk = int(my_chunk//1) # we get the integer part
print (my_chunk)
# create the iterator
iter_csv = pd.read_csv(
PATH,
iterator=True,
chunksize=my_chunk)
# concatenate according to a filter to our result dataframe
df_result = pd.concat(
[chunk[chunk[‘n3’]>0]
for chunk in iter_csv])
print (df_result)