[librecat-dev] Catmandu(?) eating memory

[Michael Kapp]

Hello Nicolas,

thanks for the quick reply.

I will do some more tests with the following modified script using a 
reference for the store (and mysql), hope that will do the trick...

thx & regards,
Michael

---------------------------------

package Cat::DB;
use Mouse;
use Catmandu::Store::DBI;
use namespace::autoclean;
use Data::Dumper;

has 'data' => (
   traits    => ['Hash'],
   is        => 'rw',
   required  => 1,
   isa       => 'HashRef',
   default   => sub { {} },
);

has 'store' => (
   is        => 'ro',
   isa       => 'Catmandu::Store::DBI',
);

sub create {
   my $self = shift;
   eval {
     $self->store->add($self->data); ## (was) eating memory
   };
   if (my $e = $@) {
     print("Something went wrong: $e\n");
     return;
   }
};

no Mouse;
__PACKAGE__->meta->make_immutable;
1;

package main;
use strict;
use warnings;
use Data::Dumper;
use Catmandu::Store::DBI;

my $start_time = time();
my $wait_time = 1;
my $never_happen = 0;
my $seconds = 0;

my $store = Catmandu::Store::DBI->new(
     data_source => 'DBI:mysql:database=test',
     username => 'test',
     password => 'secret',
     bags => {
       test => {
         autocommit => 1,
         plugins => [ 'Datestamps' ]
       }
     }
);

do {
     if (((time() - $start_time) % $wait_time) == 0) {
         &create_data;
     }
} until ($never_happen);

sub create_data {
     print "Seconds elapsed: $seconds\n";
     my $test_dat = { map { rand($_) => 1 } 1 .. 1000 };
     my $sqlite = Cat::DB->new(data => $test_dat, store => $store );
     $sqlite->create;
     sleep(1); # to stop multiple instances
     $seconds += $wait_time;
}




On 09/03/16 12:45, Nicolas Steenlant wrote:
> Hello Michael,
>
> there have been bugs with DBD::Sqlite not disconnecting if autocommit is true and there are still active statements.
> With the large number of clients you were creating, that may have been the cause of the leak.
> See for example: http://www.perlmonks.org/?node_id=665714.
>
> Regards,
> Nicolas
>
> --
> Nicolas Steenlant
> Ghent University Library
>
> Sint-Hubertusstraat 8
> 9000 Gent
>
> nicolas.steenlant at ugent.be
> +32 486 62 62 27
>
>
>
>
>> On 09 Mar 2016, at 12:39, Nicolas Steenlant <nicolas.steenlant at ugent.be> wrote:
>>
>> Hello Michael,
>>
>> you are creating a new store and sqlite client with each call.
>> It would be better to keep a reference to the store in your Cat::DB instance (has => ’store’ …).
>>
>> Regards,
>> Nicolas
>>
>> --
>> Nicolas Steenlant
>> Ghent University Library
>>
>> Sint-Hubertusstraat 8
>> 9000 Gent
>>
>> nicolas.steenlant at ugent.be
>> +32 486 62 62 27
>>
>>
>>
>>
>>> On 09 Mar 2016, at 12:17, Michael Kapp <limodev at yulux.de> wrote:
>>>
>>> Hi,
>>>
>>> the below script eating all memory.
>>>
>>> Every two seconds it creates a hash (with test data) and put's it into a catmandu sqlite db:
>>>
>>> my $test_dat = { map { rand($_) => 1 } 1 .. 1000 };
>>> my $sqlite = Cat::DB->new(data => $test_dat);
>>> $sqlite->create;
>>>
>>> If I comment the line:
>>>
>>> $self->env->add($self->data); ## eating memory
>>>
>>> The possible memory leak stops.
>>>
>>> I don't know what I'm doing wrong.
>>>
>>> Are there any hints to prevent this?
>>>
>>> thx & regards,
>>> Michael
>>>
>>> ---------------------------------
>>>
>>> package Cat::DB;
>>> use Mouse;
>>> use Catmandu::Store::DBI;
>>> use namespace::autoclean;
>>>
>>> has 'data' => (
>>> traits    => ['Hash'],
>>> is        => 'rw',
>>> required  => 1,
>>> isa       => 'HashRef',
>>> default   => sub { {} },
>>> );
>>>
>>> sub env {
>>> my $self = shift;
>>> my $cstore = Catmandu::Store::DBI->new(
>>>        data_source => 'DBI:SQLite:database=/tmp/test.db',
>>>        bags => {
>>>          test => {
>>>            autocommit => 1,
>>>            plugins => [ 'Datestamps' ]
>>>          }
>>>        }
>>>      );
>>> return $cstore->bag('test');
>>> };
>>>
>>> sub create {
>>> my $self = shift;
>>> eval {
>>>    $self->env->add($self->data); ## eating memory
>>> };
>>> if (my $e = $@) {
>>>      print("Something went wrong: $e\n");
>>>      return;
>>> }
>>> };
>>>
>>> no Mouse;
>>> __PACKAGE__->meta->make_immutable;
>>> 1;
>>>
>>> package main;
>>> use strict;
>>> use warnings;
>>>
>>> my $start_time = time();
>>> my $wait_time = 2;
>>> my $never_happen = 0;
>>> my $seconds = 0;
>>>
>>> do {
>>>    if (((time() - $start_time) % $wait_time) == 0) {
>>>        &create_data;
>>>    }
>>> } until ($never_happen);
>>>
>>>
>>> sub create_data {
>>>    print "Seconds elapsed: $seconds\n";
>>>    my $test_dat = { map { rand($_) => 1 } 1 .. 1000 };
>>>    my $sqlite = Cat::DB->new(data => $test_dat);
>>>    $sqlite->create;
>>>    sleep(1); # to stop multiple instances
>>>    $seconds += $wait_time;
>>> }
>>>
>>>
>>> _______________________________________________
>>> librecat-dev mailing list
>>> - send list mails to librecat-dev at lists.uni-bielefeld.de
>>> - to unsubscribe or change options, visit https://lists.uni-bielefeld.de/mailman2/cgi/unibi/listinfo/librecat-dev
>>> - project website: http://librecat.org/
>>
>