For creating custom form fields please use the following steps:

Step 1:Add all the custom fields to the following XML file components/com_library/views/book/default.xml

<fields name="request" addfieldpath="administrator/components/com_library/models/fields">
    <fieldset name="request">
        <field 
            name="id" 
            type="modal_book" 
            label="COM_LIBRARY_BOOK_FIELD_ID_LABEL" 
            description="COM_LIBRARY_BOOK_FIELD_ID_DESC"
            required="true"
        />
    </fieldset>
</fields>

Step 2: Create class with custom form field name administrator/components/com_library/models/fields/modal/book.php. // No direct access defined('_JEXEC') or die('Restricted access');

jimport('joomla.form.formfield');

/** * Book form field class */

<?php class JFormFieldModal_Book extends JFormField
 {
     /**
      * field type
      * @var string
      */
     protected $type = 'Modal_Book';


 Override parent class getinput method.
 protected function getInput()
  {
        /* This function will get all the custom fields value from database and create the HTML
            and then that HTML will be passed to the view

        */
             // Load modal behavior
      JHtml::_('behavior.modal', 'a.modal');

      // Build the script
      $script = array();
      $script[] = '    function jSelectBook_'.$this->id.'(id, title, object) {';
      $script[] = '        document.id("'.$this->id.'_id").value = id;';
      $script[] = '        document.id("'.$this->id.'_name").value = title;';
      $script[] = '        SqueezeBox.close();';
      $script[] = '    }';

      // Add to document head
      JFactory::getDocument()->addScriptDeclaration(implode("\n", $script));

      // Setup variables for display
      $html = array();
      $link = 'index.php?option=com_library&amp;view=books&amp;layout=modal'.
                  '&amp;tmpl=component&amp;function=jSelectBook_'.$this->id;

      $db = JFactory::getDbo();
      $query = $db->getQuery(true);
      $query->select('title');
      $query->from('#__books');
      $query->where('id='.(int)$this->value);
      $db->setQuery($query);
      if (!$title = $db->loadResult()) {
          JError::raiseWarning(500, $db->getErrorMsg());
      }
      if (empty($title)) {
          $title = JText::_('COM_LIBRARY_FIELD_SELECT_BOOK');
      }
      $title = htmlspecialchars($title, ENT_QUOTES, 'UTF-8');

      // The current book input field
      $html[] = '<div class="fltlft">';
      $html[] = '  <input type="text" id="'.$this->id.'_name" value="'.$title.'" disabled="disabled" size="35" />';
      $html[] = '</div>';

      // The book select button
      $html[] = '<div class="button2-left">';
      $html[] = '  <div class="blank">';
      $html[] = '    <a class="modal" title="'.JText::_('COM_LIBRARY_SELECT_BOOK_TITLE').'" href="'.$link.
                         '" rel="{handler: \'iframe\', size: {x:800, y:450}}">'.
                         JText::_('COM_LIBRARY_BUTTON_SELECT_BOOK').'</a>';
      $html[] = '  </div>';
      $html[] = '</div>';

      // The active book id field
      if (0 == (int)$this->value) {
          $value = '';
      } else {
          $value = (int)$this->value;
      }

      // class='required' for client side validation
      $class = '';
      if ($this->required) {
          $class = ' class="required modal-value"';
      }

      $html[] = '<input type="hidden" id="'.$this->id.'_id"'.$class.' name="'.$this->name.'" value="'.$value.'" />';

      return implode("\n", $html);
  }
}

Step 3:Create Modal layout: Create the following file /administrator/components/com_library/views/books/tmpl/modal.php.

/

/ No direct access
 defined('_JEXEC') or die('Restricted access');

 // Load tooltip behavior
 JHtml::_('behavior.tooltip');
 $listOrder    = $this->escape($this->state->get('list.ordering'));
 $listDirn    = $this->escape($this->state->get('list.direction'));

 $function = JFactory::getApplication()->input->getCmd('function', 'jSelectBook');
?>
<form action="<?php echo $this->action; ?>" method="post" name="adminForm" id="adminForm">
<table class="adminlist">
    <thead>
        <tr>
            <th>
                <?php echo JHtml::_('grid.sort', 'COM_LIBRARY_BOOKS_HEADING_TITLE', 'title', $listDirn, $listOrder); ?>
            </th>
            <th>
                <?php echo JHtml::_('grid.sort', 'COM_LIBRARY_BOOKS_HEADING_STATE', 'state', $listDirn, $listOrder); ?>
            </th>
            <th>
                <?php echo JHtml::_('grid.sort', 'COM_LIBRARY_BOOKS_HEADING_HITS', 'hits', $listDirn, $listOrder); ?>
            </th>
            <th>
                <?php echo JHtml::_('grid.sort', 'COM_LIBRARY_BOOKS_HEADING_CREATED', 'created', $listDirn, $listOrder); ?>
            </th>
            <th>
                <?php echo JHtml::_('grid.sort', 'COM_LIBRARY_BOOKS_HEADING_MODIFIED', 'modified', $listDirn, $listOrder); ?>
            </th>
            <th width="5">
                <?php echo JHtml::_('grid.sort', 'COM_LIBRARY_BOOKS_HEADING_ID', 'id', $listDirn, $listOrder); ?>
            </th>
        </tr>
    </thead>
    <tfoot>
        <tr>
            <td colspan="6"><?php echo $this->pagination->getListFooter(); ?></td>
        </tr>
    </tfoot>
    <tbody>
<?php foreach ($this->items as $i => $item) : ?>
        <tr class="row<?php echo $i % 2; ?>">
            <td>
                <a class="pointer" onclick="if (window.parent) window.parent.<?php echo $this->escape($function);?>('<?php echo $item->id; ?>', '<?php echo $this->escape(addslashes($item->title)); ?>');"><?php echo $this->escape($item->title); ?></a>
            </td>
            <td align="center"><?php echo JHtml::_('jgrid.published', $item->state, $i, 'books.'); ?></td>
            <td align="right"><?php echo $item->hits; ?></td>
            <td align="center"><?php echo $item->created; ?></td>
            <td align="center"><?php echo $item->modified; ?></td>
            <td><?php echo $item->id; ?></td>
        </tr>
<?php endforeach; ?>
    </tbody>
</table>
<div>
    <input type="hidden" name="task" value="" />
    <input type="hidden" name="boxchecked" value="0" />
    <input type="hidden" name="filter_order" value="<?php echo $listOrder; ?>" />
    <input type="hidden" name="filter_order_Dir" value="<?php echo $listDirn; ?>" />
    <?php echo JHtml::_('form.token'); ?>
</div>
</form>