Tuesday, January 29, 2013

Using PHP's auto_prepend_file to log all server errors

We recently updated our PHP configuration to move from our applications manually logging errors to having it be done at a server-wide level by taking advantage of PHP's auto_prepend_file feature.

This allows you to catch all errors reported by PHP and ship them off to some sort of logging application.

Below is the ServerLogger class that we used to accomplish this. It'll probably make its way to GitHub eventually but that's a task for another day. To implement this, follow the instructions in the class comments below.

/**
 * This library is meant to used in a auto_prepend_file to enable server-wide
 * tracking of PHP errors.
 * 
 * Installation (You may use your own paths, these are examples):
 * 
 *   - Place ServerLogger.php in /srv/www/server-scripts/ServerLogger.php
 * 
 *   - Create auto_prepend.php /srv/www/server-scripts/auto_prepend.php with:
 * 
 *      
 * @see https://github.com/Graylog2/gelf-php
 */

if(defined('SERVER_LOGGER')) {
  return;
}

define('SERVER_LOGGER', 1);

/**
 * @see https://github.com/Graylog2/gelf-php 
 */
class GELFMessage
{
  /**
   * @var string
   */
  private $version = null;

  /**
   * @var integer
   */
  private $timestamp = null;

  /**
   * @var string
   */
  private $shortMessage = null;

  /**
   * @var string
   */
  private $fullMessage = null;

  /**
   * @var string
   */
  private $facility = null;

  /**
   * @var string
   */
  private $host = null;

  /**
   * @var integer
   */
  private $level = null;

  /**
   * @var string
   */
  private $file = null;

  /**
   * @var integer
   */
  private $line = null;

  /**
   * @var array
   */
  private $data = array();

  /**
   * @param string $version
   * @return GELFMessage
   */
  public function setVersion($version)
  {
    $this->version = $version;
    return $this;
  }

  /**
   * @return string
   */
  public function getVersion()
  {
    return $this->version;
  }

  /**
   * @param integer $timestamp
   * @return GELFMessage
   */
  public function setTimestamp($timestamp)
  {
    $this->timestamp = $timestamp;
    return $this;
  }

  /**
   * @return integer
   */
  public function getTimestamp()
  {
    return $this->timestamp;
  }

  /**
   * @param string $shortMessage
   * @return GELFMessage
   */
  public function setShortMessage($shortMessage)
  {
    $this->shortMessage = $shortMessage;
    return $this;
  }

  /**
   * @return string
   */
  public function getShortMessage()
  {
    return $this->shortMessage;
  }

  /**
   * @param string $fullMessage
   * @return GELFMessage
   */
  public function setFullMessage($fullMessage)
  {
    $this->fullMessage = $fullMessage;
    return $this;
  }

  /**
   * @return string
   */
  public function getFullMessage()
  {
    return $this->fullMessage;
  }

  /**
   * @param string $facility
   * @return GELFMessage
   */
  public function setFacility($facility)
  {
    $this->facility = $facility;
    return $this;
  }

  /**
   * @return string
   */
  public function getFacility()
  {
    return $this->facility;
  }

  /**
   * @param string $host
   * @return GELFMessage
   */
  public function setHost($host)
  {
    $this->host = $host;
    return $this;
  }

  /**
   * @return string
   */
  public function getHost()
  {
    return $this->host;
  }

  /**
   * @param integer $level
   * @return GELFMessage
   */
  public function setLevel($level)
  {
    $this->level = $level;
    return $this;
  }

  /**
   * @return integer
   */
  public function getLevel()
  {
    return $this->level;
  }

  /**
   * @param string $file
   * @return GELFMessage
   */
  public function setFile($file)
  {
    $this->file = $file;
    return $this;
  }

  /**
   * @return string
   */
  public function getFile()
  {
    return $this->file;
  }

  /**
   * @param integer $line
   * @return GELFMessage
   */
  public function setLine($line)
  {
    $this->line = $line;
    return $this;
  }

  /**
   * @return integer
   */
  public function getLine()
  {
    return $this->line;
  }

  /**
   * @param string $key
   * @param mixed $value
   * @return GELFMessage
   */
  public function setAdditional($key, $value)
  {
    $this->data["_" . trim($key)] = $value;
    return $this;
  }

  /**
   * @return mixed
   */
  public function getAdditional($key)
  {
    return isset($this->data["_" . trim($key)]) ? $this->data[$key] : null;
  }

  /**
   * @return array
   */
  public function toArray()
  {
    $messageAsArray = array(
        'version' => $this->getVersion(),
        'timestamp' => $this->getTimestamp(),
        'short_message' => $this->getShortMessage(),
        'full_message' => $this->getFullMessage(),
        'facility' => $this->getFacility(),
        'host' => $this->getHost(),
        'level' => $this->getLevel(),
        'file' => $this->getFile(),
        'line' => $this->getLine(),
    );

    foreach ($this->data as $key => $value) {
      $messageAsArray[$key] = $value;
    }

    return $messageAsArray;
  }

}

/**
 * @see https://github.com/Graylog2/gelf-php 
 */
class GELFMessagePublisher
{
  /**
   * @var integer
   */

  const CHUNK_SIZE_WAN = 1420;

  /**
   * @var integer
   */
  const CHUNK_SIZE_LAN = 8154;

  /**
   * @var integer
   */
  const GRAYLOG2_DEFAULT_PORT = 12201;

  /**
   * @var string
   */
  const GRAYLOG2_PROTOCOL_VERSION = '1.0';

  /**
   * @var string
   */
  protected $hostname = null;

  /**
   * @var integer
   */
  protected $port = null;

  /**
   * @var integer
   */
  protected $chunkSize = null;

  /**
   * Creates a new publisher that sends errors to a Graylog2 server via UDP
   *
   * @throws InvalidArgumentException
   * @param string $hostname
   * @param integer $port
   * @param integer $chunkSize
   */
  public function __construct($hostname, $port = self::GRAYLOG2_DEFAULT_PORT, $chunkSize = self::CHUNK_SIZE_WAN)
  {
    // Check whether the parameters are set correctly
    if (!$hostname) {
      throw new InvalidArgumentException('$hostname must be set');
    }

    if (!is_numeric($port)) {
      throw new InvalidArgumentException('$port must be an integer');
    }

    if (!is_numeric($chunkSize)) {
      throw new InvalidArgumentException('$chunkSize must be an integer');
    }

    $this->hostname = $hostname;
    $this->port = $port;
    $this->chunkSize = $chunkSize;
  }

  /**
   * Publishes a GELFMessage, returns false if an error occured during write
   *
   * @throws UnexpectedValueException
   * @param unknown_type $message
   * @return boolean
   */
  public function publish(GELFMessage $message)
  {
    // Check if required message parameters are set
    if (!$message->getShortMessage() || !$message->getHost()) {
      throw new UnexpectedValueException(
              'Missing required data parameter: "version", "short_message" and "host" are required.'
      );
    }

    // Set Graylog protocol version
    $message->setVersion(self::GRAYLOG2_PROTOCOL_VERSION);

    // Encode the message as json string and compress it using gzip
    $preparedMessage = $this->getPreparedMessage($message);

    // Open a udp connection to graylog server
    $socket = $this->getSocketConnection();

    // Several udp writes are required to publish the message
    if ($this->isMessageSizeGreaterChunkSize($preparedMessage)) {
      // A unique id which consists of the microtime and a random value
      $messageId = $this->getMessageId();

      // Split the message into chunks
      $messageChunks = $this->getMessageChunks($preparedMessage);
      $messageChunksCount = count($messageChunks);

      // Send chunks to graylog server
      foreach (array_values($messageChunks) as $messageChunkIndex => $messageChunk) {
        $bytesWritten = $this->writeMessageChunkToSocket(
                $socket, $messageId, $messageChunk, $messageChunkIndex, $messageChunksCount
        );

        if (false === $bytesWritten) {
          // Abort due to write error
          return false;
        }
      }
    } else {
      // A single write is enough to get the message published
      if (false === $this->writeMessageToSocket($socket, $preparedMessage)) {
        // Abort due to write error
        return false;
      }
    }

    // This increases stability a lot if messages are sent in a loop
    // A value of 20 means 0.02 ms
    usleep(20);

    // Message successful sent
    return true;
  }

  /**
   * @param GELFMessage $message
   * @return string
   */
  protected function getPreparedMessage(GELFMessage $message)
  {
    return gzcompress(json_encode($message->toArray()));
  }

  /**
   * @return resource
   */
  protected function getSocketConnection()
  {
    return stream_socket_client(sprintf('udp://%s:%d', gethostbyname($this->hostname), $this->port));
  }

  /**
   * @param string $preparedMessage
   * @return boolean
   */
  protected function isMessageSizeGreaterChunkSize($preparedMessage)
  {
    return (strlen($preparedMessage) > $this->chunkSize);
  }

  /**
   * @return float
   */
  protected function getMessageId()
  {
    return (float) (microtime(true) . mt_rand(0, 10000));
  }

  /**
   * @param string $preparedMessage
   * @return array
   */
  protected function getMessageChunks($preparedMessage)
  {
    return str_split($preparedMessage, $this->chunkSize);
  }

  /**
   * @param float $messageId
   * @param string $data
   * @param integer $sequence
   * @param integer $sequenceSize
   * @throws InvalidArgumentException
   * @return string
   */
  protected function prependChunkInformation($messageId, $data, $sequence, $sequenceSize)
  {
    if (!is_string($data) || $data === '') {
      throw new InvalidArgumentException('Data must be a string and not be empty.');
    }

    if (!is_integer($sequence) || !is_integer($sequenceSize)) {
      throw new InvalidArgumentException('Sequence number and size must be integer.');
    }

    if ($sequenceSize <= 0) {
      throw new InvalidArgumentException('Sequence size must be greater than 0.');
    }

    if ($sequence > $sequenceSize) {
      throw new InvalidArgumentException('Sequence size must be greater than sequence number.');
    }

    return pack('CC', 30, 15) . substr(md5($messageId, true), 0, 8) . pack('CC', $sequence, $sequenceSize) . $data;
  }

  /**
   * @param resource $socket
   * @param float $messageId
   * @param string $messageChunk
   * @param integer $messageChunkIndex
   * @param integer $messageChunksCount
   * @return integer|boolean
   */
  protected function writeMessageChunkToSocket($socket, $messageId, $messageChunk, $messageChunkIndex, $messageChunksCount)
  {
    return fwrite(
                    $socket, $this->prependChunkInformation($messageId, $messageChunk, $messageChunkIndex, $messageChunksCount)
    );
  }

  /**
   * @param resource $socket
   * @param string $preparedMessage
   * @return integer|boolean
   */
  protected function writeMessageToSocket($socket, $preparedMessage)
  {
    return fwrite($socket, $preparedMessage);
  }

}

class ServerLogger
{
  public static function configure($hostname, $port = GELFMessagePublisher::GRAYLOG2_DEFAULT_PORT, $chunkSize = GELFMessagePublisher::CHUNK_SIZE_WAN)
  {
    if(self::$instance !== null) {
      throw new RuntimeException('You can only call configure() once.');
    }
    
    $publisher = new GELFMessagePublisher($hostname, $port, $chunkSize);
    
    self::$instance = new ServerLogger($publisher);
  }
  
  public static function configureSettings(array $settings)
  {
    if(isset($settings['additional'])) {
      foreach($settings['additional'] as $key => $value) {
        self::$settings['additional'][$key] = $value;
      }
      
      unset($settings['additional']);
    }
    
    foreach($settings as $key => $value) {
      self::$settings[$key] = $value;
    }
  }

  public static function logException(Exception $exception)
  {
    if(!self::$instance) {
      throw new InvalidArgumentException('Must call ServerLogger::configure() first.');
    }
    
    self::$instance->listenToException($exception);
  }
  
  public function listenToError($errno , $errstr, $errfile = '', $errline = 0, array $errcontext = array())
  {
    if(!self::$settings['enable']) {
      return;
    }
    
    if($this->errorsHandled < self::$settings['maximum_errors']) {
      $this->errorsHandled++;
      
      if(!self::$settings['report_ignored'] && error_reporting() === 0) {
        return;
      }
      
      $message = $this->getBaseMessage();
      $message->setShortMessage('error: '.$errstr);
      
      $message->setAdditional('error_number', $errno);
      
      if($errfile) {
        $message->setFile($errfile);
      }
      
      if($errline) {
        $message->setLine($errline);
      }
      
      if(isset($_SESSION, $_REQUEST)) {
        $message->setAdditional('session_and_request', array_merge_recursive($_SESSION, $_REQUEST));
      }
      
      $this->publisher->publish($message);
    }
  }
  
  public function listenToException(Exception $exception)
  {
    if(!self::$settings['enable']) {
      return;
    }
    
    $message = $this->getBaseMessage();
    $message->setShortMessage('exception: '.$exception->getMessage());
    $message->setFile($exception->getFile());
    $message->setLine($exception->getLine());
    $message->setFullMessage($exception->getTraceAsString());
    $message->setAdditional('exception_class', get_class($exception));
    $message->setAdditional('exception_code', $exception->getCode());

    $this->publisher->publish($message);
  }
  
  public function listenToShutdown()
  {
    if(!self::$settings['enable']) {
      return;
    }
    
    $this->reservedMemory = null;
    
    if(($error = error_get_last()) && !$this->wasReported($error)) {
      $message = $this->getBaseMessage();
      $message->setShortMessage('error: '.$error['message']);
      
      unset($error['message']);

      foreach($error as $k => $v) {
        $message->setAdditional($k, $v);
      }

      $this->publisher->publish($message);
    }
  }
  
  /**
   * @return GELFMessage 
   */
  private function getBaseMessage()
  {
    $message = new GELFMessage();
    
    if($host = gethostname()) {
      $message->setHost($host);
    }
    
    $message->setFacility(self::$settings['facility']);
    
    foreach(self::$settings['additional'] as $k => $v) {
      $message->setAdditional($k, $v);
    }
    
    if(isset($_GET)) {
      $message->setAdditional('$_GET', json_encode($_GET));
    }
    
    if(isset($_POST)) {
      $message->setAdditional('$_POST', json_encode($_POST));
    }
    
    foreach(self::$settings['additional_server_keys'] as $k) {
      $server = array();

      if(isset($_SERVER[$k])) {
        $server[$k] = $_SERVER[$k];
      }

      if(count($server)) {
        $message->setAdditional('$_SERVER (abridged)', json_encode($server));
      }
    }
    
    return $message;
  }
  
  private function wasReported($error)
  {
    // 417 = bitmask: E_ERROR + E_CORE_ERROR + E_COMPILE_ERROR + E_USER_ERROR
    
    return $this->errorsHandled > 0 && ($error['type'] & 417) > 0;
  }
  
  private function __construct(GELFMessagePublisher $publisher)
  {
    $this->publisher = $publisher;
    $this->reservedMemory = str_repeat('1', self::$settings['reserved_memory']);
    
    set_error_handler(array($this, 'listenToError'));
    
    set_exception_handler(array($this, 'listenToException'));
    
    register_shutdown_function(array($this, 'listenToShutdown'));
  }
  
  private static $settings = array(
    'facility'                => 'php server logger',
    'enable'                  => true,
    'maximum_errors'          => 100,
    'reserved_memory'         => 524288,
    'report_ignored'          => false,
    'additional'              => array(),
    'additional_server_keys'  => array('HTTP_HOST', 'HTTP_X_FORWARDED_HOST', 'HTTP_X_FORWARDED_PROTOCOL', 'REQUEST_URI')
  ); 
  
  private static $instance = null;
  private $publisher;
  private $reservedMemory = null;
  private $errorsHandled = 0;
}

Thursday, March 15, 2012

Generating V5 UUIDs in MySQL

I am working on a project that requires me to generate version 5 (v5) UUIDs in MySQL. Unfortunately, MySQL's UUID() only generates v1 UUIDs. Furthermore, I didn't find any information online so I was forced to develop my own solution.

It turns out that it's actually a bit of a monster (so it would be better to wrap it in a function) but I figure the raw SELECT statement should be enough to help out anyone that's looking to do such a thing. The idea behind v5 UUID is that you sha1 hash the concatenation of a namespace and an identifier, truncate the result to 128 bits, and then set the particular bits to signify the version and variant. v5 UUIDs should always be the same given the same namespace and identifier so that makes it fairly easy to write up a test suite against a working implementation.

You can substitute the namespace UUID with binary data, and the 'www.example.org' with more relevant data - such as a column name in the current result set or similar. I simply used the two ones below as an example.

Note - There's some trickiness here because you need to do some bit arithmetic on a 128 bit result, and MySQL only supports 64 bit arithmetic. To get around this, I split the calculation and bitmask into two 64 bit parts and concatenated the results together. Also, version 3 would be very similar -- just use different bitmasks and md5.

Update 11/16/2012 - I added the function definition for UUID_V5().
DELIMITER $$

CREATE FUNCTION UUID_V5(namespace CHAR(36), object_key TEXT)
RETURNS CHAR(36) DETERMINISTIC
BEGIN
  DECLARE hex CHAR(32);
  
  SET hex = CONCAT(
      CONV(
        CAST(CONV(HEX(LEFT(UNHEX(SHA1(CONCAT(UNHEX(REPLACE(namespace, '-', '')), object_key))), 8)), 16, 10) AS UNSIGNED)
          &
        CAST(18446744073709490175 AS UNSIGNED)
          |
        CAST(20480 AS UNSIGNED),
      10, 16),
      CONV(
        CAST(CONV(HEX(RIGHT(LEFT(UNHEX(SHA1(CONCAT(UNHEX(REPLACE(namespace, '-', '')), object_key))), 16), 8)), 16, 10) AS UNSIGNED)
          &
        CAST(4611686018427387903 AS UNSIGNED)
          |
        CAST(9223372036854775808 AS UNSIGNED),
      10, 16)
  );
  
  RETURN LOWER(CONCAT(LEFT(hex, 8), '-', MID(hex, 9,4), '-', MID(hex, 13,4), '-', MID(hex, 17,4), '-', RIGHT(hex, 12)));
END

$$

DELIMITER ;
SELECT UNHEX(CONCAT(
  CONV(
    CAST(CONV(HEX(LEFT(UNHEX(SHA1(CONCAT(UNHEX(REPLACE('6ba7b810-9dad-11d1-80b4-00c04fd430c8', '-', '')), 'www.example.org'))), 8)), 16, 10) AS UNSIGNED)
      &
    CAST(18446744073709490175 AS UNSIGNED)
      |
    CAST(20480 AS UNSIGNED),
  10, 16),
  CONV(
    CAST(CONV(HEX(RIGHT(LEFT(UNHEX(SHA1(CONCAT(UNHEX(REPLACE('6ba7b810-9dad-11d1-80b4-00c04fd430c8', '-', '')), 'www.example.org'))), 16), 8)), 16, 10) AS UNSIGNED)
      &
    CAST(4611686018427387903 AS UNSIGNED)
      |
    CAST(9223372036854775808 AS UNSIGNED),
  10, 16)
))

Wednesday, November 16, 2011

XenServer, Encrypted Local Storage and SMART monitoring

We are upgrading our XenServer machines to 6.0 and I decided to look into doing proper SMART monitoring on the hard drives.

I also wanted to put out a guide on setting up encrypted local storage for virtual machines. This will satisfy "data encryption at rest" type regulations such as HIPAA without having to encrypt each and every VM.

This has been tested on both XenServer 5.6 SP2 and 6.0. For demonstration purposes, the machine has two hard drives - one used to install XenServer, and one used solely for encrypted virtual machine space.
  1. Obtain XenServer ISO and begin installation
  2. When prompted for which hard drive to use, only use the first listed. Leave the other unchecked.
  3. Enter the usuals for hostname, IP address, DNS, NTP etc.
  4. After the installer has finished, reboot and SSH into the new installation.
  5. Find the path to the hard drive you wish to use for encrypted storage. Suppose for guide purposes this is /dev/sdb
    $ fdisk -
  6. Encrypt the device -
    $ cryptsetup luksFormat /dev/sdb
  7. Open the device -
    $ cryptsetup luksOpen /dev/sdb data
  8. Setup crypttab to automatically open the device on boot
    $ echo "data /dev/sdb" > /etc/crypttab
  9. Edit bootloader config so you can see the encryption passphrase prompt. You will be removing the words quiet and splash from the first entry. If you wish, you can skip this step, but you must use CTRL+ALT+F2 to switch to the proper virtual console when XenServer is booting.
    $ vi /boot/extlinux.conf
  10. Obtain the host UUID (uuid (R): xxxxxxx-xxxx-xxx.....)
    $ xe host-list
  11. Create a new storage repository (Be sure to replace $host_uuid with the UUID from the previous step)
    $ # I am fighting with the code highlighting script so ignore this line : )
    
    $ xe sr-create content-type=user device-config:device=/dev/mapper/data host-uuid=$host_uuid name-label="Local Storage - Encrypted" shared=false type=lvm
    
  12. Install packages for sending emails and SMART monitoring.
    $ cd ~
    $ wget http://vault.centos.org/5.4/os/i386/CentOS/mailx-8.1.1-44.2.2.i386.rpm
    $ wget http://vault.centos.org/5.4/os/i386/CentOS/smartmontools-5.38-2.el5.i386.rpm
    $ rpm -hiv smartmontools-5.38-2.el5.i386.rpm mailx-8.1.1-44.2.2.i386.rpm
    $ vi /etc/ssmtp/ssmtp.conf # See below for file content
    $ vi /etc/ssmtp/revaliases # See below for file content
    $ vi /etc/smartd.conf # See below for file content
    $ /etc/init.d/smartd restart
    
  13. Reboot and enter the encryption key on startup. Everything should be working now. You should get a test email indicating that smart monitoring is working everytime the machine starts up.
  14. Using a Windows computer, connect to the machine in XenCenter. Verify that everything looks okay. Note the Local Storage - Encrypted" entry. Right click it and select "Set as Default." This will allow future VMs to be created on encrypted storage without explicitly picking a disk.
  15. If the host you installed on has two network cards, you can bond them. Select the NICs tab in XenCenter and then the "Create Bond" button. Select both NICs and "Automatically add this network to the new virtual machines." 

/etc/ssmtp/ssmtp.conf
root=xxx@gmail.com
mailhub=smtp.gmail.com:587
hostname=xxx@gmail.com
UseTLS=YES
UseSTARTTLS=YES
AuthMethod=LOGIN
AuthUser=xxx@gmail.com
AuthPass=xxx
FromLineOverride=YES
/etc/ssmtp/revaliases

# sSMTP aliases
#
# Format:       local_account:outgoing_address:mailhub
#
# Example: root:your_login@your.domain:mailhub.your.domain[:port]
# where [:port] is an optional port number that defaults to 25.

root:xxx@gmail.com:smtp.gmail.com:587
/etc/smartd.conf

# List all devices to monitor here

# This does a short monitoring every day between 02:00 and 03:00 and a long test
# every Saturday between 03:00 and 04:00.

# -M test sends out a test email to make sure emailing is working when
# the daemon starts up

/dev/sda -d ata -s (S/../.././02|L/../../6/03) -t -m xxx@gmail.com -M test
/dev/sdb -d ata -s (S/../.././02|L/../../6/03) -t -m xxx@gmail.com

Tuesday, October 18, 2011

Streaming content with Symfony 1.4

Often times, you have to stream content in web applications. Unfortunately, the design of the view system in symfony 1 assumes content is built entirely (In memory) and then sent to the client. To get around this, we've done some bad hacks historically to send headers and then forced output buffering off.

Using PHP 5.3 closures (Although, 5.2 callbacks will work, its just the syntax is less nice), we're able to have sfWebResponse execute code when it sends the content, rather than just echoing an already built string.

These snippets are rather raw and contrived, but you should be able to extract the idea from them if you need the functionality in your own project. It assumes you have your own local response class (via factories.yml) and your own mix-in class to extend sfComponent (And thus, sfActions).

The example code will print Number 0..9 with a 1 second delay between prints. Your browser should show each number as it is flushed.

someActions
class someActions extends sfActions
{
  public function executeTestCallback()
  {
    $start = 1;

    return $this->renderCallable(function() use($start) {
      for($i = 0; $i < 10; $i++) {
        echo "The number is $i", PHP_EOL;

        flush();

        sleep(1);
      }
    });
  }
}
sfComponentExtension (Configuration)
class sfComponentExtension
{
  ...

  /**
   * Allows an action to send code that will render the response when called
   * rather than returning a pre-rendered string. This allows content rendering
   * to be deferred until sendContent() is called, which can result in less
   * memory usage if you are streaming files or similar.
   * 
   * Example usage - (Snipped)
   * 
   * @param callable $callable
   * @return int
   */
  public function renderCallable($callable)
  {
    $this->component->getResponse()->setContent($callable);

    return sfView::NONE;
  }

  ...
}
sfProjectWebResponse (Configuration)
class sfProjectWebResponse extends sfWebResponse
{
  ...

   /**
   * Sends headers and content. Responsible for executing content if it is
   * a callable.
   */
  public function sendContent()
  {
    if(is_string($this->content)) {
      parent::sendContent();
    } else if(is_callable($this->content)) {
      // Copied below from parent classes to keep behavior consistent
      
      if(!$this->headerOnly) {
        // no such thing as filtering content for callables, code left below
        // for reference
        //  $event = $this->dispatcher->filter(new sfEvent($this, 'response.filter_content'), $this->getContent());
        //  $content = $event->getReturnValue();

        if($this->options['logging']) {
          $this->dispatcher->notify(new sfEvent($this, 'application.log', array('Send content (callable o)')));
        }
        
        while(ob_get_level()) {
          ob_end_clean();
        }
        
        flush();
        
        call_user_func($this->content);
      }
    }
  }

  ...
}



P.S. I'm looking at packaging a lot of these snippets into something more useful. We use a lot of other additions to Symfony 1.4 at work and are locked into it with migration to 2.0 not an option, so I have been toying around with the idea of forking sf1.4 to implement new functionality and remove some poor (And slow) design decisions.

Monday, May 23, 2011

Symfony 1.4 and subdirectories for models, forms and filters

Symfony 1.4 with Doctrine stores all models in lib/model/doctrine, all forms in lib/form/doctrine and all filters in lib/filter/doctrine. This is a good behavior for small projects but can get out of hand when you have lots of tables as the folders get really large.

While there is an option "package" there are a few problems with its implementation. First, it is intended to be used for internal plugin implementation. As such, it introduces intermediate plugin classes even when you aren't packaging your models as a plugin. Second, it doesn't apply to form or filter classes.

Below is a patch to allow symfony to honor a new option, "directory" which will build these three items in sub-directories instead. As with all Doctrine schema options, you can apply this option in individual schema.yml files to all models or to only specific ones.

I just developed these mods this morning so there may be some issues with them. Ad-hoc testing seems to indicate that they work with at least the Doctrine features we use in our codebase : )

The patch
Index: lib/plugins/sfDoctrinePlugin/lib/generator/sfDoctrineFormGenerator.class.php
===================================================================
--- lib/plugins/sfDoctrinePlugin/lib/generator/sfDoctrineFormGenerator.class.php (revision 6558)
+++ lib/plugins/sfDoctrinePlugin/lib/generator/sfDoctrineFormGenerator.class.php (working copy)
@@ -93,6 +93,10 @@
       $this->modelName = $model;
 
       $baseDir = sfConfig::get('sf_lib_dir') . '/form/doctrine';
+      
+      if($dirOption = $this->table->getOption('directory')) {
+        $baseDir .= '/'.$dirOption;
+      }
 
       $isPluginModel = $this->isPluginModel($model);
       if ($isPluginModel)
Index: lib/plugins/sfDoctrinePlugin/lib/generator/sfDoctrineFormFilterGenerator.class.php
===================================================================
--- lib/plugins/sfDoctrinePlugin/lib/generator/sfDoctrineFormFilterGenerator.class.php (revision 6558)
+++ lib/plugins/sfDoctrinePlugin/lib/generator/sfDoctrineFormFilterGenerator.class.php (working copy)
@@ -76,6 +76,10 @@
       $this->modelName = $model;
 
       $baseDir = sfConfig::get('sf_lib_dir') . '/filter/doctrine';
+      
+      if($dirOption = $this->table->getOption('directory')) {
+        $baseDir .= '/'.$dirOption;
+      }
 
       $isPluginModel = $this->isPluginModel($model);
       if ($isPluginModel)
Index: lib/plugins/sfDoctrinePlugin/lib/vendor/doctrine/Doctrine/Import/Builder.php
===================================================================
--- lib/plugins/sfDoctrinePlugin/lib/vendor/doctrine/Doctrine/Import/Builder.php (revision 6558)
+++ lib/plugins/sfDoctrinePlugin/lib/vendor/doctrine/Doctrine/Import/Builder.php (working copy)
@@ -1189,7 +1189,13 @@
                 $definition['inheritance']['extends'] = $prefix . $definition['inheritance']['extends'];
             }
         }
-
+        
+        $__path = $this->_path;
+        
+        if(isset($definition['options']['directory'])) {
+          $this->_path = sprintf('%s/%s', $this->getTargetPath(), $definition['options']['directory']);
+        }
+        
         $definitionCode = $this->buildDefinition($definition);
 
         if ($prefix) {
@@ -1199,7 +1205,7 @@
         }
 
         $fileName = $this->_getFileName($originalClassName, $definition);
-
+        
         $packagesPath = $this->_packagesPath ? $this->_packagesPath:$this->_path;
 
         // If this is a main class that either extends from Base or Package class
@@ -1274,6 +1280,8 @@
         } else {
             $bytes = file_put_contents($writePath, $code);
         }
+        
+        $this->_path = $__path;
 
         if (isset($bytes) && $bytes === false) {
             throw new Doctrine_Import_Builder_Exception("Couldn't write file " . $writePath);


Applying to a specific model in a schema.yml file (Any .yml file in config/doctrine)

User:
  options:
    directory: prevention
  tableName: xxx
  columns:
    ..
UserAccount:
  ..

Applying to all models in a schema.yml file (Any .yml file in config/doctrine)

options:
  directory: prevention

User:
  tableName: xxx
  columns:
    ..
UserAccount:
  ..

Wednesday, December 8, 2010

Symfony 1.4 & Doctrine UUID plugin

I implemented a simple plugin that can generate v4/v5 UUIDs for Doctrine models.

You can grab a copy of it here. Any feedback or comments are welcome. :)

Wednesday, November 24, 2010

Symfony and per-module routing.yml

Symfony's routing system is instantiated by settings in the project and application routing.yml. This is a nice design, but it can be difficult to manage routing with many modules. The routing file can easily approach hundreds or thousands of lines after some modest development work, and its difficult to group together related routes. Furthermore, having to edit a centralized routing.yml file everytime a new module or action is added to the project violates separation of concerns and modularity.

A quick glance at the structure of a symfony project reveals that each module has a config directory. However, since routing.yml is what translates the request uri into a module and action, this file (among a few others) can not benefit from this configuration separation.

I've worked out a pretty simple solution to this problem in the form of parsing out all available modules and loading the routing.yml (and other configurable files) explicitly. This is also compatible with Fabien's recommended method of creating links between different applications.

ProjectConfiguration
class ProjectConfiguration extends sfProjectConfiguration
{
  ...

  protected function getModulesWithGlobalConfig($applicationName, $config)
  {
    $directory = sprintf('%s/%s/modules', sfConfig::get('sf_apps_dir'), $applicationName);
    $files = array();

    foreach(sfFinder::type('dir')->maxdepth(0)->in($directory) as $modulePath) {
      if(file_exists($file = $modulePath.'/'.$config)) {
        $files[] = $file;
      }
    }

    return $files;
  }

  protected function getMergedConfigPaths($configPath, $applicationName, $paths)
  {
    if(in_array($configPath, sfConfig::get('sf_global_config_paths'))) {
      $paths = array_merge($this->getModulesWithGlobalConfig($applicationName, $configPath), $paths);
    }

    return $paths;
  }
}

With the above methods, this only requires one more modification to each application that you wish to enable this feature for inside your project.

applicationNameConfiguration
class applicationNameConfiguration extends sfApplicationConfiguration
{
  ...

  /**
   * Merges global configs with this applications configuration.
   * 
   * This provides this application with e.g. routing.yml for each module.
   *
   * @param string $configPath
   * @return array
   */
  public function getConfigPaths($configPath)
  {
    return $this->getMergedConfigPaths($configPath, 'applicationName', parent::getConfigPaths($configPath));
  }
}

If you wish for cross application links to stay compatible, a tweak will be needed there as well.
ProjectConfiguration::getApplicationRouting()
class ProjectConfiguration extends sfProjectConfiguration
{
  ...

  private function getApplicationRouting($applicationName)
  {
    if(!isset($this->applicationRouting[$applicationName])) {
      $this->applicationRouting[$applicationName] = new sfPatternRouting(new sfEventDispatcher());

      $config = new sfRoutingConfigHandler();

      $routingPath = 'config/routing.yml';

      $routingPaths = $this->getMergedConfigPaths($routingPath, $applicationName, array(
        sprintf('%s/%s/%s', sfConfig::get('sf_apps_dir'), $applicationName, $routingPath)
      ));

      $routes = $config->evaluate($routingPaths);

      $this->applicationRouting[$applicationName]->setRoutes($routes);
    }

    return $this->applicationRouting[$applicationName];
  }
}


With the above, all thats needed is a simple sfConfig statement to configure which configuration files should be searched for in all module directories.

Wherever appropriate, i.e. project or application configure()
sfConfig::set('sf_global_config_paths', array(
  'config/routing.yml'
));

If you followed the above, you should be able to use multiple routing.yml files in your project. This could be further improved by having "sf_global_config_paths" define a map to the new config names. For example, array('config/routing.yml' => 'config/global_routing.yml'). This would help infer that the config file is processed on every request.

In terms of performance, a proper caching layer should be thrown infront of getModulesWithGlobalConfig() so that the search for modules and config files is only done when project:cache-clear is run, but I will leave that exercise up to the reader : )