This should to the work you were so close.

CREATE OR REPLACE FUNCTION tmp_tzk1_f_preslozi_string (str IN VARCHAR)
RETURN string
IS 
    znak CHAR := '';
    len NUMBER;
    brojac_svako_trece INTEGER := 0;
    str1 VARCHAR(20) := '';

BEGIN
    len := LENGTH(str);
    FOR i IN REVERSE 1..len
    LOOP
        brojac_svako_trece := brojac_svako_trece +1;
        znak := SUBSTR (str, i, 1);
        IF MOD(brojac_svako_trece,3) = 0 THEN
            brojac_svako_trece := 0;
            IF ASCII(znak)<97 THEN
                znak := LOWER(znak);
            ELSE 
                znak := UPPER(znak);
            END IF;
        END IF;
        str1 := str1 || znak;
    END LOOP;
    RETURN str1;
END;

DECLARE
   RetVal   VARCHAR2 (32767);
BEGIN
    
   RetVal := TMP_TZK1_F_PRESLOZI_STRING('DobarDan');
   
   dbms_output.put_line(RetVal);
END;

This should do the work you were so close.

CREATE OR REPLACE FUNCTION tmp_tzk1_f_preslozi_string (str IN VARCHAR)
RETURN string
IS 
    znak CHAR := '';
    len NUMBER;
    brojac_svako_trece INTEGER := 0;
    str1 VARCHAR(20) := '';

BEGIN
    len := LENGTH(str);
    FOR i IN REVERSE 1..len
    LOOP
        brojac_svako_trece := brojac_svako_trece +1;
        znak := SUBSTR (str, i, 1);
        IF MOD(brojac_svako_trece,3) = 0 THEN
            brojac_svako_trece := 0;
            IF ASCII(znak)<97 THEN
                znak := LOWER(znak);
            ELSE 
                znak := UPPER(znak);
            END IF;
        END IF;
        str1 := str1 || znak;
    END LOOP;
    RETURN str1;
END;

DECLARE
   RetVal   VARCHAR2 (32767);
BEGIN
    
   RetVal := TMP_TZK1_F_PRESLOZI_STRING('DobarDan');
   
   dbms_output.put_line(RetVal);
END;