Doctrine batch inserting uses 2GB of Ram

Question

I am trying to insert approximately 200k of records with Symfony2 and Doctrine via Doctrine Fixtures Bundle. I am using flush and clear but at the end script uses 1.8 GB of RAM.

This is the class that loads SmartMeter entities into database:

<?php

namespace HTEC\SmartMeteringAPIBundle\DataFixtures\ORM;

use Doctrine\Common\DataFixtures\AbstractFixture;
use Doctrine\Common\DataFixtures\FixtureInterface;
use Doctrine\Common\Persistence\ObjectManager;
use Doctrine\Common\DataFixtures\OrderedFixtureInterface;

use HTEC\SmartMeteringAPIBundle\Entity\SmartMeter;

use Symfony\Component\DependencyInjection\ContainerAwareInterface;
use Symfony\Component\DependencyInjection\ContainerInterface;

class LoadSmartMeterData extends AbstractFixture implements FixtureInterface, ContainerAwareInterface, OrderedFixtureInterface
{

    static $NUMBER_OF_SMART_METERS = 0;
    static $MAX_NUM_OF_SM_PER_CONC = 500;

     /**
     * @var ContainerInterface
     */
    private $container;


    public function getOrder()
    {
        return 10; // the order in which fixtures will be loaded
    }

    /**
     * @inheritDoc
     */
    public function setContainer(ContainerInterface $container = null)
    {
        $this->container = $container;
    }

    /**
     * {@inheritDoc}
     */
    public function load(ObjectManager $manager)
    {

        $numberOfUsers = LoadUserData::$NUMBER_OF_USERS;
        $numberOfConcentrators = LoadConcentratorData::$NUMBER_OF_CONCENTRATORS;
        $numberOfTariffs = LoadTariffData::$NUMBER_OF_TARIFFS;

        $numberOfSmartMetersPerConcentrator = 0;

        $smartMeter = null;
        $concentrator = null;
        $user = null;
        $tariff = null;

        $feeders = array();
        $numberOfFeeders = 0;
        $feedersRandomKey = array();

        $isIpAddressDynamic = false;

        $numberOfTransformerFeeders = 0;

        $smartMeterType = 0;
        $smartMeterProtocol = 0;
        $modemType = 0;

        $numberOfSmartMeters = 0;


        echo "\n\nCreating Smart Meters. This could take a couple of minutes and could take approximately 2 GB of RAM \n\n...";

        $startTime = time();

        $smartMeterCount = 0;

        $lastChunkNumber = 0;
        $currentChunkNumber = 0;

        $lastSmartMeterNumber = 0;
        $smartMetersClearedCurrently = 0;

        $concentratorSmartMeters = null;

        for($i = 0; $i < LoadConcentratorData::$NUMBER_OF_CONCENTRATORS; $i++)
        {

            $concentrator = $manager->getRepository('SMAPIBundle:Concentrator')->find($i+1);

            $numberOfSmartMetersPerConcentrator = rand(1, self::$MAX_NUM_OF_SM_PER_CONC);

            $numberOfSmartMeters += $numberOfSmartMetersPerConcentrator;

            for($c = 0; $c < $numberOfSmartMetersPerConcentrator; $c++)
            {
                $smartMeter = new SmartMeter();
                $smartMeter->setSerialNumber(++$smartMeterCount);
                $smartMeter->setConcentrator($concentrator);

                $smartMeterType = rand(1,3);

                switch($smartMeterType)
                {
                    case 1:
                        $smartMeter->setType('DIRECT');
                    break;

                    case 2:
                        $smartMeter->setType('HALF');
                    break;

                    case 3:
                        $smartMeter->setType('INDIRECT');
                    break;
                }

                $user = $manager->getRepository('SMAPIBundle:User')->find(rand(1, $numberOfUsers));
                $smartMeter->setCreatedBy($user);

                $numberOfTransformerFeeders = $concentrator->getTransformerFeeders()->count();
                $feeders = $concentrator->getTransformerFeeders()->toArray();

                if($numberOfTransformerFeeders > 0)
                {

                    $feedersRandomKey = array_rand($feeders, 1);

                    if(isset($feeders[$feedersRandomKey]))
                    {
                        $smartMeter->setFeeder($feeders[$feedersRandomKey]);
                    }
                }

                $smartMeter->setStatus(rand(0,2));


                $tariff = $manager->getRepository('SMAPIBundle:Tariff')->find(rand(1, $numberOfTariffs));
                $smartMeter->setTariff($tariff);

                $smartMeterProtocol = rand(1,3);

                switch($smartMeterProtocol)
                {
                    case 1:
                        $smartMeter->setProtocol('DLMS');
                    break;

                    case 2:
                        $smartMeter->setProtocol('EURIDIS');
                    break;

                    case 3:
                        $smartMeter->setProtocol('IEC');
                    break;
                }


                $smartMeter->setModemSerialNumber(rand(1, 9000000000));

                $smartMeter->setManufacture('MAN: ' . ($i + $c));

                $modemType = rand(1,2);

                if($modemType === 1)
                {
                    $smartMeter->setModemType('PLC');
                    $smartMeter->setModemIndex(rand(1, 512));
                }
                else
                {

                    $smartMeter->setModemType('GPRS');
                    $isIpAddressDynamic = rand(0,1);

                    if($isIpAddressDynamic === 1)
                    {
                        $smartMeter->setModemIpAddress(rand(1000000, 2000000));
                    }
                    else
                    {
                        $smartMeter->setModemPhoneNumber($this->getRandomIpAddressV4());
                    }
                }


                if(rand(0,1) === 1)
                {
                    $smartMeter->setModemRepeaterNumber(rand(10000000, 90000000));
                }

                $manager->persist($smartMeter);

            } // end of FOR numberOfSmartMetersPerConcentrator

            // flush smart meters
            $manager->flush();
            $manager->clear();

            $currentChunkNumber = ceil($smartMeterCount / 5000);

            if($smartMeterCount > 5000 && $currentChunkNumber > $lastChunkNumber)
            {
                $lastChunkNumber = $currentChunkNumber;



                if($lastSmartMeterNumber > 0)
                {
                    $smartMetersClearedCurrently = $smartMeterCount - $lastSmartMeterNumber;
                }
                else
                {
                    $smartMetersClearedCurrently = $smartMeterCount;
                }


                echo "\n\nFlushing and clearing " . number_format($smartMetersClearedCurrently, 0, ',', '.') . " Smart Meters.\nTotal memory used after flush and clear: " . number_format(((memory_get_usage(true) / 1024) / 1024), 2, ',', '.') . " Megabytes\n\n...";

                $lastSmartMeterNumber = $smartMeterCount;
            }
        }// end for NUMBER_OF_CONCENTRATORS

        $manager->flush();
        $manager->clear();

        unset($concentrator);
        unset($smartMeter);
        unset($tariff);

        echo "\n\n-------------------------------------------\n\n";
        echo "\n\nTotal memory used after final flush and clear of Smart Meters: " . number_format(((memory_get_usage(true) / 1024) / 1024), 2, ',', '.') . " Megabytes\n\n...";

        echo "\n\nCreating Smart Meters complete. Created " . number_format($numberOfSmartMeters, 0, ',', '.') . " Smart Meters.\n\n";

        $durationSeconds = (time() - $startTime);
        $durationMinutes = $durationSeconds / 60;

        $secondsRemainder = $durationSeconds % 60;

        echo "\n\nTotal duration time: " . ceil($durationMinutes) . " minutes and " . $secondsRemainder . " seconds.\n\n\n";

        self::$NUMBER_OF_SMART_METERS = $numberOfSmartMeters;

    }

    public function getRandomIpAddressV4()
    {
        return rand(1, 255) . '.' . rand(0, 255) . '.' . rand(0, 255) . '.' . rand(0, 255);
    }
}

On approximately 5000 records I am calling flush and clear but it seams that no memory is released.

Are there recommendations about how to prevent Doctrine from using too much RAM during batch tasks?

Answer 1

If it is a command, try running it with the option --no-debug. Or, you can disable the logger by calling $manager->getConnection()->getConfiguration()->setSQLLogger(null); at the beginning of your loadfunction.

Either way disabling the logger saves quite a lot of memory during Doctrine batch tasks.