Simple Brainfuck interpreter in C with support for nested subroutines

Advice 1

#define MIN_MEMORY_SIZE 30000

Later...

static void init(BrainfuckState *bf, size_t size)
{
    assert(size <= MIN_MEMORY_SIZE);
    ...
}

Actually, it's a maximum memory size. I would rename it to MAX_TAPE_LENGTH.

Advice 2

typedef struct
{
    uint8_t *memory;
    size_t pointer;
} BrainfuckState;

I would add a memory capacity and rename memory to tape, since, in Brainfuck, the "memory" is not random-access, but rather access data sequentially.

Advice 3

static void init(BrainfuckState *bf, size_t size)
{
    assert(size <= MIN_MEMORY_SIZE);
    bf->memory = calloc(size, 1);
    assert(!(bf == NULL));
    bf->pointer = 0;
}

I would move the check assert(!(bf == NULL)); to the first statement in init and change it to assert(bf).

Advice 4

static void deinit(BrainfuckState *bf)
{
    free(bf->memory);
    bf->pointer = 0;
}

I would change it to

static void deinit(BrainfuckState* bf)
{
    free(bf->tape);
    bf->tape = NULL;
    bf->tape_length = 0;
    bf->pointer = 0;
}

Advice 5

case '>':
    bf->pointer++;
    break;

case '<':
    bf->pointer--;
    break;

May overflow. I would protect like this:

case '>':
    if (bf->pointer < bf->tape_length - 1) {
        bf->pointer++;
    }
    break;

case '<': 
    if (bf->pointer > 0) {
        bf->pointer--;
    }    
    break;

Advice 6

Before executing some Brainfuck code, I would check that the square brackets are balanced. For example,

++[[>][>]]

is balanced. And

++][]]

is not. For that end, you need a stack data structure. The check algorithm looks like:

stack st;
for ch in source:
    switch ch {
        case '[':
            st.push(ch)
            break
        case ']':
            if st is empty:
                return false
            st.pop()
            break
    }
return st is empty

Summa summarum

All in all, I had this rewrite in mind:

#define _CRT_SECURE_NO_WARNINGS /* Visual Studio 2022 specific */

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <assert.h>

#define MAX_TAPE_LENGTH 30000

typedef struct char_stack_node {
    struct char_stack_node* prev;
    char ch;
} char_stack_node;

typedef struct char_stack {
    char_stack_node* top;
    size_t size;
} char_stack;

void char_stack_init(char_stack* st) {
    st->size = 0;
    st->top = NULL;
}

void char_stack_push(char_stack* st, char ch) {
    char_stack_node* node = malloc(sizeof *node);
    node->ch = ch;
    node->prev = st->top;
    st->top = node;
    st->size++;
}

int char_stack_is_empty(char_stack* st) {
    return st->size == 0;
}

char char_stack_pop(char_stack* st) {
    char datum;
    char_stack_node* old_top = st->top;
    datum = old_top->ch;
    st->top = st->top->prev;
    st->size--;
    free(old_top);
    return datum;
}

static int check_is_parentheses_balanced(const char *const source, 
                                         size_t source_length) {
    char_stack st;
    char_stack_init(&st);

    for (size_t i = 0; i < source_length; ++i) {
        char ch = source[i];

        switch (ch) {
            case '[':
                char_stack_push(&st, ch);
                break;

            case ']':
                if (char_stack_is_empty(&st)) {
                    return 0;
                }

                char_stack_pop(&st);
                break;
        }
    }

    return char_stack_is_empty(&st) ? 1 : 0;
}

typedef struct
{
    uint8_t* tape;
    size_t pointer;
    size_t tape_length;
} BrainfuckState;

static void init(BrainfuckState* bf, size_t tape_length)
{
    assert(bf);
    assert(tape_length <= MAX_TAPE_LENGTH);
    bf->tape = calloc(tape_length, sizeof *bf->tape);
    bf->tape_length = tape_length;
    bf->pointer = 0;
}

static void deinit(BrainfuckState* bf)
{
    free(bf->tape);
    bf->tape = NULL;
    bf->tape_length = 0;
    bf->pointer = 0;
}

static void interpret(BrainfuckState* bf, const char* source, size_t source_length)
{
    size_t unmatchedBracketCount = 0;

    if (!check_is_parentheses_balanced(source, source_length)) {
        fprintf(stderr, "Error: unbalanced square brackets.\n");
        exit(EXIT_FAILURE);
    }

    for (size_t i = 0; i < source_length; i++)
    {
        switch (source[i])
        {
        case '.':
            printf("%c", bf->tape[bf->pointer]);
            break;

        case ',':
            scanf("%c", &(bf->tape[bf->pointer]));
            break;

        case '+':
            bf->tape[bf->pointer]++;
            break;

        case '-':
            bf->tape[bf->pointer]--;
            break;

        case '>':

            if (bf->pointer < bf->tape_length - 1) {
                bf->pointer++;
            }

            break;

        case '<':
            
            if (bf->pointer > 0) {
                bf->pointer--;
            }
            
            break;

        case '[':
            if (bf->tape[bf->pointer] == 0)
            {
                unmatchedBracketCount++;

                while (source[i] != ']' || unmatchedBracketCount)
                {
                    i++;

                    if (source[i] == '[')
                        unmatchedBracketCount++;
                    else if (source[i] == ']')
                        unmatchedBracketCount--;
                }
            }
            break;

        case ']':
            if (bf->tape[bf->pointer])
            {
                unmatchedBracketCount++;

                while (source[i] != '[' || unmatchedBracketCount)
                {
                    i--;

                    if (source[i] == ']')
                        unmatchedBracketCount++;
                    else if (source[i] == '[')
                        unmatchedBracketCount--;
                }
            }
            break;
        }
    }
}

int main(int argc, char** argv)
{
    if (argc < 2)
        fprintf(stderr, "Pass a filename as a command line arguement.\n");
    else
    {
        FILE* fp = fopen(argv[1], "r");
        assert(fp);

        fseek(fp, 0, SEEK_END);
        size_t sourceSize = ftell(fp);
        char* source = malloc(sourceSize);
        assert(source);
        rewind(fp);

        for (size_t i = 0; i < sourceSize; i++)
            source[i] = fgetc(fp);

        fclose(fp);

        BrainfuckState bf;
        init(&bf, MAX_TAPE_LENGTH);
        interpret(&bf, source, sourceSize);
        deinit(&bf);

        free(source);
    }

    return EXIT_SUCCESS;
}

```