[](https://wesmckinney.com/blog/arrow-streaming-columnar/#cb1-1)import time
[](https://wesmckinney.com/blog/arrow-streaming-columnar/#cb1-2)import numpy as np
[](https://wesmckinney.com/blog/arrow-streaming-columnar/#cb1-3)import pandas as pd
[](https://wesmckinney.com/blog/arrow-streaming-columnar/#cb1-4)import pyarrow as pa
[](https://wesmckinney.com/blog/arrow-streaming-columnar/#cb1-5)
[](https://wesmckinney.com/blog/arrow-streaming-columnar/#cb1-6)def generate_data(total_size, ncols):
[](https://wesmckinney.com/blog/arrow-streaming-columnar/#cb1-7)    nrows = int(total_size / ncols / np.dtype('float64').itemsize)
[](https://wesmckinney.com/blog/arrow-streaming-columnar/#cb1-8)    return pd.DataFrame({
[](https://wesmckinney.com/blog/arrow-streaming-columnar/#cb1-9)        'c' + str(i): np.random.randn(nrows)
[](https://wesmckinney.com/blog/arrow-streaming-columnar/#cb1-10)        for i in range(ncols)
[](https://wesmckinney.com/blog/arrow-streaming-columnar/#cb1-11)    })