Parsing a large text file with Python

The main performance bottle neck is appending to a dataframe all the time. Instead, you could create a data buffer and expand that buffer once it overflows. The code below generates a synthetic dataset of about 100,000 lines of data, and then parses the corresponding data file:

import pandas as pd
import numpy as np
from itertools import combinations_with_replacement
from scipy.misc import comb
from time import time

np.random.seed(0)

# Array buffer increment size
array_size = 1000
# Data file (output and input)
filename = "stack_output.dat"


def generate_data(m):
    """Generate synthetic (dummy) data to test performance"""

    # Weird string appearing in the example data
    sep_string = "".join(["o--"]*26)
    sep_string += "o\n"

    # Generate ZSC data, which seem to be combinatoric in nature
    x = np.arange(m)
    Ngroups = comb(m, 3, exact=True, repetition=True)

    # For each group of ZSC, generate a random number of lines of data
    # (between 2 and 8 lines)
    Nrows = np.random.randint(low=2, high=8, size=Ngroups)

    # Open file and write data
    with open(filename, "w") as f:
        # Loop over all values of ZSC (000, 001, 010, 011, etc.)
        for n, ZSC in enumerate(combinations_with_replacement(x, 3)):
            # Generate random data
            rand_data = np.random.rand(Nrows[n], 5)
            # Write (meta) data to file
            f.write(sep_string)
            f.write("Z_%d  S_%d  C_%d\n" % ZSC)
            f.write("foo    bar\n")
            f.write("X[m]   Y[m]   Z[m]   alpha[-]   beta[-]\n")
            for data in rand_data:
                f.write("%.8e  %.8e  %.8e  %.8e  %.8e\n" % tuple(data))

    return True


def grow_array(x):
    """Helper function to expand an array"""
    buf = np.zeros((array_size, x.shape[1])) * np.nan
    return np.vstack([x, buf])


def parse_data():
    """Parse the data using a growing buffer"""

    # Number of lines of meta data (i.e. line that don't
    # contain the XYZ alpha beta values
    Nmeta = 3

    # Some counters
    Ndata = 0
    group_index = 0

    # Data buffer
    all_data = np.zeros((array_size, 8)) * np.nan

    # Read filename
    with open(filename, "r") as f:
        # Iterate over all lines
        for i, line in enumerate(f):
            # If we're at that weird separating line, we know we're at the
            # start of a new group of data, defined by Z, S, C
            if line[0] == "o":
                group_index = i
            # If we're one line below the separator, get the Z, S, C values
            elif i - group_index == 1:
                ZSC = line.split()
                # Extract the number from the string
                Z = ZSC[0][2:]
                S = ZSC[1][2:]
                C = ZSC[2][2:]
                ZSC_clean = np.array([Z, S, C])
            # If we're in a line below the meta data, extract the XYZ values
            elif i - group_index > Nmeta:
                # Split the numbers in the line
                data = np.array(line.split(), dtype=float)
                # Check if the data still fits in buffer.
                # If not: expand the buffer
                if Ndata == len(all_data)-1:
                    all_data = grow_array(all_data)
                # Populate the buffer
                all_data[Ndata] = np.hstack([ZSC_clean, data])
                Ndata += 1
    # Convert the buffer to a pandas dataframe (and clip the unpopulated
    # bits of the buffer, which are still NaN)
    df = pd.DataFrame(all_data, columns=("Z", "S", "C", "X", "Y", "Z", "alpha", "beta")).dropna(how="all")
    return df

t0 = time()
generate_data(50)
t1 = time()
data = parse_data()
t2 = time()

print("Data size: \t\t\t %i" % len(data))
print("Rendering data: \t %.3e s" % (t1 - t0))
print("Parsing data: \t\t %.3e s" % (t2 - t1))

Result:

Data size:           99627
Rendering data:      3.360e-01 s
Parsing data:        1.356e+00 s

Is this good enough for your purposes?

Previous answer for reference (which assumed certain structure of the data file):

You can use the skiprows feature in pandas.read_csv. In your example, only the last 3 lines of each multiple of 9 contain useful data, so you can use skiprows along with a function that returns True if the line index is 6, 7, or 8 (starting at 0) for each multiple of 9:

import pandas as pd
filename = "data.dat"

data = pd.read_csv(
    filename, names=("X", "Y", "Z", "alpha", "beta"), delim_whitespace=True,
    skiprows=lambda x: x % 9 < 6,
)
print(data)