AngularJS performance problems

I'm building a complex angular app which has infinity scrolling with marsonry-like columns and images. I did a research and I writted my own directive to render the data for me:

angular.module('deepsy.flexgrid', []);

angular.module('deepsy.flexgrid').directive('flexgrid', [

    'flexgridFactory',

    function initialize(flexgridFactory) {
        return flexgridFactory.create();
    }
]);


angular.module('deepsy.flexgrid').factory('flexgridFactory', [

    'FlexGrid',
    '$log',

    function initialize(FlexGrid, $log) {

        function Creator() {
            this.restrict = 'AE';

            this.scope = {
                'model': '=source',
                'opts': '=options'
            };

            this.$$grid = null;

            this.link = this.$$link.bind(this);
        }

        Creator.prototype.$$link = function $$link(scope, elem, attrs) {
            scope.itemTemplate = attrs.template;

            scope.mother = scope.$parent;

            scope.template = elem.html();
            elem.html('');

            this.$$grid = new FlexGrid(scope, elem);
        };

        return {
            create: function create() {
                return new Creator();
            }
        };
    }
]);

angular.module('deepsy.flexgrid').factory('FlexGrid', [

    '$compile',

    function($compile) {

        function FlexGrid(scope, elem) {
            this.scope = scope;
            this.elem = elem;
            this.state = null;
            this.counter = 0;
            this.id = Math.floor(Math.random() * 999);

            this.$getColumns();
            this.$createColumns();
            window.onresize = this.$watchSize.bind(this);


            this.scope.$watch(function() {
                return this.scope.model;
            }.bind(this), this.$applyData.bind(this), true);
        }

        FlexGrid.prototype.$getColumns = function() {
            var curr = null,
                width = document.body.clientWidth,
                min, max;

            for (var i in this.scope.opts.columns) {
                curr = this.scope.opts.columns[i];
                min = curr.min || 0;
                max = curr.max || 99999;

                if (min < width && width < max) {
                    this.state = curr;
                    return curr;
                }
            }
        };

        FlexGrid.prototype.$createColumns = function() {
            var output = [];
            for (var i = 0; i < this.state.columns; i++) {
                output.push('<div id="flexgrid_' + this.id + '_' + i + '" class="gridColumns ' + this.state.class + '"></div>');
            }
            this.elem.html(output.join(''));
        };

        FlexGrid.prototype.$watchSize = function() {
            var curr = this.state || {
                columns: 0
            };
            this.$getColumns();
            if (this.state.columns != curr.columns) {
                this.$createColumns();
                this.$fillData(0);
            }
        };

        FlexGrid.prototype.$applyData = function(newVal, oldVal) {

            var bindings = [],
                count = 0;

            oldVal.forEach(function(obj) {
                bindings.push(obj._id);
            });

            newVal.forEach(function(obj) {
                if (bindings.indexOf(obj._id) != -1) {
                    count++;
                }
            });

            if (count == oldVal.length && oldVal.length > 0) {
                this.$fillData(count);
                //  console.log('add');
            } else {
                this.$fillData(0);
                //  console.log('render');
            }
        };

        FlexGrid.prototype.$fillData = function(start) {
            var columns = this.state.columns,
                len = this.scope.model.length;

            if (start === 0) {
                this.$clearColumns(columns);
                this.counter = 0;
            }

            for (var i = start; i < len; i++) {
                $("#flexgrid_" + this.id + "_" + this.counter).append(this.$compile(this.scope.model[i]));

                if (this.counter++ === columns - 1)
                    this.counter = 0;
            }

            //$("img", this.elem).load(function(){
            //  $(this).parent().fadeIn(700);
            //});
        };

        FlexGrid.prototype.$compile = function(data) {
            var compiled = this.scope.template.replace(/\{\|[^\|\}]*\|\}/gmi, function(exp, val) {
                return data[exp.replace(/\|\}|\{\|/gmi, '')];
            });

            compiled = compiled.replace('src-image', 'src="' + data.image + '" dinimg');

            return compiled;
        };

        FlexGrid.prototype.$clearColumns = function(columns) {
            for (var j = 0; j < columns; j++) {
                $("#flexgrid_" + this.id + "_" + j).empty();
            }
        };

        return FlexGrid;
    }
]);

It's working awesome, but there are some performance issues. I was firstly like "hmm.. I may have too many watchers maybe", but then I optimised my code and after executing the following code in the browser console:

(function () { 
var root = $(document.getElementsByTagName('body'));
var watchers = [];

var f = function (element) {
    if (element.data().hasOwnProperty('$scope')) {
        angular.forEach(element.data().$scope.$$watchers, function (watcher) {
            watchers.push(watcher);
        });
    }

    angular.forEach(element.children(), function (childElement) {
        f($(childElement));
    });
};

f(root);

console.log(watchers.length);
})();

I figured out that I got only 1 watcher! So the watchers might not be the problem. With this code I have implemented an infinity scrolling. Each element cotains image and text. The problem is that after getting over 200-250 items, my UI starts to lag, even If I have 1 watcher. I thought it can be because of the image size, but after I executed in the Chrome console $(".item").html('') and cleared the content of all boxes my UI is still not smooth. The problem is not caused by too many DOM elements too because I got only 38 divs total beside the divs rendered by the directive. How I can increase the performance of my app?

Edit: Also a weird thing that I noticed if that even if I remove all elements from the DOM(after rendering them) via $(".item").remove, the browser still uses 300mb+ RAM.