Looking at the help caller text, the first word of the returned text is the line number.
We can use the bash read command to split it into words:
read lineno rest <<< "$(caller)"
Using a regex is not so onerous
[[ $(caller) =~ ^([[:digit:]]+) ]] && lineno=${BASH_REMATCH[1]}
#!/bin/bash
                                          # line 2
a() { b; }                                # line 3
b() { c; }                                # line 4
c() {                                     # line 5
    caller                                # line 6
    out=$(caller); echo "$out"            # line 7
    i=0                                   # line 8
    while true; do                        # line 9
        c=$(caller $i)                    # line 10
        [[ -z "$c" ]] && break            # line 11
        printf '%d = %s\n' $((i++)) "$c"  # line 12
    done                                  # line 13
}                                         # line 14
a                                         # line 15
outputs
4 caller.sh
4 caller.sh
0 = 4 b caller.sh
1 = 3 a caller.sh
2 = 15 main caller.sh