src/org/objectweb/cjdbc/controller/loadbalancer/raidb2/RAIDb2.java

説明を見る。

package org.objectweb.cjdbc.controller.loadbalancer.raidb2;

import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;

import org.objectweb.cjdbc.common.exceptions.BadConnectionException;
import org.objectweb.cjdbc.common.exceptions.UnreachableBackendException;
import org.objectweb.cjdbc.common.i18n.Translate;
import org.objectweb.cjdbc.common.log.Trace;
import org.objectweb.cjdbc.common.sql.AbstractWriteRequest;
import org.objectweb.cjdbc.common.sql.ParsingGranularities;
import org.objectweb.cjdbc.common.sql.SelectRequest;
import org.objectweb.cjdbc.common.sql.StoredProcedure;
import org.objectweb.cjdbc.common.util.ReadPrioritaryFIFOWriteLock;
import org.objectweb.cjdbc.common.xml.DatabasesXmlTags;
import org.objectweb.cjdbc.controller.backend.DatabaseBackend;
import org.objectweb.cjdbc.controller.cache.metadata.MetadataCache;
import org.objectweb.cjdbc.controller.connection.AbstractConnectionManager;
import org.objectweb.cjdbc.controller.loadbalancer.AbstractLoadBalancer;
import org.objectweb.cjdbc.controller.loadbalancer.AllBackendsFailedException;
import org.objectweb.cjdbc.controller.loadbalancer.BackendWorkerThread;
import org.objectweb.cjdbc.controller.loadbalancer.policies.WaitForCompletionPolicy;
import org.objectweb.cjdbc.controller.loadbalancer.policies.createtable.CreateTableException;
import org.objectweb.cjdbc.controller.loadbalancer.policies.createtable.CreateTablePolicy;
import org.objectweb.cjdbc.controller.loadbalancer.policies.createtable.CreateTableRule;
import org.objectweb.cjdbc.controller.loadbalancer.tasks.AbstractTask;
import org.objectweb.cjdbc.controller.loadbalancer.tasks.CommitTask;
import org.objectweb.cjdbc.controller.loadbalancer.tasks.KillThreadTask;
import org.objectweb.cjdbc.controller.loadbalancer.tasks.ReadStoredProcedureTask;
import org.objectweb.cjdbc.controller.loadbalancer.tasks.RollbackTask;
import org.objectweb.cjdbc.controller.loadbalancer.tasks.WriteRequestTask;
import org.objectweb.cjdbc.controller.loadbalancer.tasks.WriteRequestWithKeysTask;
import org.objectweb.cjdbc.controller.loadbalancer.tasks.WriteStoredProcedureTask;
import org.objectweb.cjdbc.controller.requestmanager.RAIDbLevels;
import org.objectweb.cjdbc.controller.requestmanager.TransactionMarkerMetaData;
import org.objectweb.cjdbc.controller.virtualdatabase.ControllerResultSet;
import org.objectweb.cjdbc.controller.virtualdatabase.VirtualDatabase;

public abstract class RAIDb2 extends AbstractLoadBalancer
{
  //
  // How the code is organized ?
  // 1. Member variables
  // 2. Constructor(s)
  // 3. Request handling
  // 4. Transaction handling
  // 5. Backend management
  //

  protected ArrayList                   backendBlockingThreads;
  protected ArrayList                   backendNonBlockingThreads;
  protected ReadPrioritaryFIFOWriteLock backendBlockingThreadsRWLock    = new ReadPrioritaryFIFOWriteLock();
  protected ReadPrioritaryFIFOWriteLock backendNonBlockingThreadsRWLock = new ReadPrioritaryFIFOWriteLock();
  // Should we wait for all backends to commit before returning ?
  protected WaitForCompletionPolicy     waitForCompletionPolicy;
  protected CreateTablePolicy           createTablePolicy;

  protected static Trace                logger                          = Trace
                                                                            .getLogger("org.objectweb.cjdbc.controller.loadbalancer.raidb2");

  /*
   * Constructors
   */

  public RAIDb2(VirtualDatabase vdb,
      WaitForCompletionPolicy waitForCompletionPolicy,
      CreateTablePolicy createTablePolicy) throws SQLException
  {
    super(vdb, RAIDbLevels.RAIDb2, ParsingGranularities.TABLE);

    this.waitForCompletionPolicy = waitForCompletionPolicy;
    backendBlockingThreads = new ArrayList();
    backendNonBlockingThreads = new ArrayList();
    this.createTablePolicy = createTablePolicy;
  }

  /*
   * Request Handling
   */

  private int getNbToWait(int nbOfThreads)
  {
    int nbToWait;
    switch (waitForCompletionPolicy.getPolicy())
    {
      case WaitForCompletionPolicy.FIRST :
        nbToWait = 1;
        break;
      case WaitForCompletionPolicy.MAJORITY :
        nbToWait = nbOfThreads / 2 + 1;
        break;
      default :
        logger
            .warn(Translate.get("loadbalancer.waitforcompletion.unsupported"));
      case WaitForCompletionPolicy.ALL :
        nbToWait = nbOfThreads;
        break;
    }
    return nbToWait;
  }

  public int execWriteRequest(AbstractWriteRequest request)
      throws AllBackendsFailedException, SQLException
  {
    return ((WriteRequestTask) execWriteRequest(request, false, null))
        .getResult();
  }

  public ControllerResultSet execWriteRequestWithKeys(
      AbstractWriteRequest request, MetadataCache metadataCache)
      throws AllBackendsFailedException, SQLException
  {
    return ((WriteRequestWithKeysTask) execWriteRequest(request, true,
        metadataCache)).getResult();
  }

  private AbstractTask execWriteRequest(AbstractWriteRequest request,
      boolean useKeys, MetadataCache metadataCache)
      throws AllBackendsFailedException, SQLException
  {
    ArrayList backendThreads;
    ReadPrioritaryFIFOWriteLock lock;

    // Handle macros
    handleMacros(request);

    // Determine which list (blocking or not) to use
    if (request.mightBlock())
    { // Blocking
      backendThreads = backendBlockingThreads;
      lock = backendBlockingThreadsRWLock;
    }
    else
    { // Non-blocking
      backendThreads = backendNonBlockingThreads;
      lock = backendNonBlockingThreadsRWLock;
      if ((waitForCompletionPolicy.getPolicy() != WaitForCompletionPolicy.ALL)
          && (request.getTransactionId() != 0))
        waitForAllWritesToComplete(request.getTransactionId());
    }

    try
    {
      lock.acquireRead();
    }
    catch (InterruptedException e)
    {
      String msg = Translate.get(
          "loadbalancer.backendlist.acquire.readlock.failed", e);
      logger.error(msg);
      throw new SQLException(msg);
    }

    int nbOfThreads = backendThreads.size();
    ArrayList writeList = new ArrayList();
    String tableName = request.getTableName();

    if (request.isCreate())
    { // Choose the backend according to the defined policy
      CreateTableRule rule = createTablePolicy.getTableRule(request
          .getTableName());
      if (rule == null)
        rule = createTablePolicy.getDefaultRule();

      // Ask the rule to pickup the backends
      ArrayList chosen;
      try
      {
        chosen = rule.getBackends(vdb.getBackends());
      }
      catch (CreateTableException e)
      {
        throw new SQLException(Translate.get(
            "loadbalancer.create.table.rule.failed", e.getMessage()));
      }

      // Build the thread list from the backend list
      if (chosen != null)
      {
        for (int i = 0; i < nbOfThreads; i++)
        {
          BackendWorkerThread thread = (BackendWorkerThread) backendThreads
              .get(i);
          if (chosen.contains(thread.getBackend()))
            writeList.add(thread);
        }
      }
    }
    else
    { // Build the list of backends that need to execute this request
      for (int i = 0; i < nbOfThreads; i++)
      {
        BackendWorkerThread thread = (BackendWorkerThread) backendThreads
            .get(i);
        if (thread.getBackend().hasTable(tableName))
          writeList.add(thread);
      }
    }

    nbOfThreads = writeList.size();
    if (nbOfThreads == 0)
    {
      String msg = Translate.get("loadbalancer.execute.no.backend.found",
          request.getSQLShortForm(vdb.getSQLShortFormLength()));
      logger.warn(msg);
      throw new SQLException(msg);
    }
    else
      logger.debug(Translate.get("loadbalancer.execute.on.several",
          new String[]{String.valueOf(request.getId()),
              String.valueOf(nbOfThreads)}));

    // Create the task
    AbstractTask task;
    if (useKeys)
      task = new WriteRequestWithKeysTask(getNbToWait(nbOfThreads),
          nbOfThreads, request, metadataCache);
    else
      task = new WriteRequestTask(getNbToWait(nbOfThreads), nbOfThreads,
          request);

    synchronized (task)
    {
      if (waitForCompletionPolicy.getPolicy() == WaitForCompletionPolicy.ALL)
      { // Post the task in each backendThread tasklist and wakeup the threads
        for (int i = 0; i < nbOfThreads; i++)
        {
          BackendWorkerThread thread = (BackendWorkerThread) backendThreads
              .get(i);
          synchronized (thread)
          {
            thread.addTask(task);
            thread.notify();
          }
        }
      }
      else
      {
        // We have to first post the request on each backend before letting the
        // first backend to execute the request. Therefore we have 2 phases:
        //   1. post the task in each thread queue
        //   2. notify each thread to execute the query

        // 1. Post the task
        if (request.mightBlock())
        {
          for (int i = 0; i < nbOfThreads; i++)
          {
            BackendWorkerThread thread = (BackendWorkerThread) backendThreads
                .get(i);
            synchronized (thread)
            {
              thread.addTask(task, request.getTransactionId());
            }
          }
        }
        else
        {
          for (int i = 0; i < nbOfThreads; i++)
          {
            BackendWorkerThread thread = (BackendWorkerThread) backendThreads
                .get(i);
            synchronized (thread)
            {
              thread.addTask(task);
            }
          }
        }

        // 2. Start the task execution on each backend
        for (int i = 0; i < nbOfThreads; i++)
        {
          BackendWorkerThread thread = (BackendWorkerThread) backendThreads
              .get(i);
          synchronized (thread)
          {
            thread.notify();
          }
        }
      }

      lock.releaseRead();

      // Wait for completion (notified by the task)
      try
      {
        // Wait on task
        long timeout = request.getTimeout() * 1000;
        if (timeout > 0)
        {
          long start = System.currentTimeMillis();
          task.wait(timeout);
          long end = System.currentTimeMillis();
          long remaining = timeout - (end - start);
          if (remaining <= 0)
          {
            String msg = Translate.get("loadbalancer.request.timeout",
                new String[]{String.valueOf(request.getId()),
                    String.valueOf(task.getSuccess()),
                    String.valueOf(task.getFailed())});

            // Try to remove the request from the task list
            lock.acquireRead();
            nbOfThreads = backendThreads.size();
            for (int i = 0; i < nbOfThreads; i++)
            {
              BackendWorkerThread thread = (BackendWorkerThread) backendThreads
                  .get(i);
              synchronized (thread)
              {
                thread.removeTask(task);
              }
            }
            lock.releaseRead();

            logger.warn(msg);
            throw new SQLException(msg);
          }
          // No need to update request timeout since the execution is finished
        }
        else
          task.wait();
      }
      catch (InterruptedException e)
      {
        // Try to remove the request from the task list
        try
        {
          lock.acquireRead();
          nbOfThreads = backendThreads.size();
        }
        catch (InterruptedException ignore)
        {
          nbOfThreads = 0; // Give up
        }
        for (int i = 0; i < nbOfThreads; i++)
        {
          BackendWorkerThread thread = (BackendWorkerThread) backendThreads
              .get(i);
          synchronized (thread)
          {
            thread.removeTask(task);
          }
        }
        lock.releaseRead();

        throw new SQLException(Translate.get("loadbalancer.request.timeout",
            new String[]{String.valueOf(request.getId()),
                String.valueOf(task.getSuccess()),
                String.valueOf(task.getFailed())}));
      }

      if (task.getSuccess() > 0)
        return task;
      else
      { // All tasks failed
        ArrayList exceptions = task.getExceptions();
        if (exceptions == null)
          throw new AllBackendsFailedException(Translate.get(
              "loadbalancer.request.failed.all", request.getId()));
        else
        {
          String errorMsg = Translate.get("loadbalancer.request.failed.stack",
              request.getId())
              + "\n";
          for (int i = 0; i < exceptions.size(); i++)
            errorMsg += ((SQLException) exceptions.get(i)).getMessage() + "\n";
          logger.error(errorMsg);
          throw new SQLException(errorMsg);
        }
      }
    }
  }

  public abstract ControllerResultSet execReadRequest(SelectRequest request,
      MetadataCache metadataCache) throws SQLException;

  protected ControllerResultSet executeRequestOnBackend(SelectRequest request,
      DatabaseBackend backend, MetadataCache metadataCache)
      throws SQLException, UnreachableBackendException
  {
    // Handle macros
    handleMacros(request);

    // Ok, we have a backend, let's execute the request
    AbstractConnectionManager cm = backend.getConnectionManager(request
        .getLogin());

    // Sanity check
    if (cm == null)
    {
      String msg = Translate.get("loadbalancer.connectionmanager.not.found",
          new String[]{request.getLogin(), backend.getName()});
      logger.error(msg);
      throw new SQLException(msg);
    }

    // Execute the query
    if (request.isAutoCommit())
    {
      if (waitForCompletionPolicy.getPolicy() != WaitForCompletionPolicy.ALL)
        // We could do something finer grain here by waiting
        // only for writes that depend on the tables we need
        // but is that really worth the overhead ?
        waitForAllWritesToComplete(backend);

      ControllerResultSet rs = null;
      boolean badConnection;
      do
      {
        badConnection = false;
        // Use a connection just for this request
        Connection c = null;
        try
        {
          c = cm.getConnection();
        }
        catch (UnreachableBackendException e1)
        {
          logger.error(Translate.get(
              "loadbalancer.backend.disabling.unreachable", backend.getName()));
          backend.disable();
          throw new UnreachableBackendException(Translate.get(
              "loadbalancer.backend.unreacheable", backend.getName()));
        }

        // Sanity check
        if (c == null)
          throw new UnreachableBackendException(
              "No more connections on backend " + backend.getName());

        // Execute Query
        try
        {
          rs = executeSelectRequestOnBackend(request, backend, c, metadataCache);
          cm.releaseConnection(c);
        }
        catch (SQLException e)
        {
          cm.releaseConnection(c);
          throw new SQLException(Translate.get(
              "loadbalancer.request.failed.on.backend", new String[]{
                  request.getSQLShortForm(vdb.getSQLShortFormLength()),
                  backend.getName(), e.getMessage()}));
        }
        catch (BadConnectionException e)
        { // Get rid of the bad connection
          cm.deleteConnection(c);
          badConnection = true;
        }
      }
      while (badConnection);
      if (logger.isDebugEnabled())
        logger.debug(Translate.get("loadbalancer.execute.on", new String[]{
            String.valueOf(request.getId()), backend.getName()}));
      return rs;
    }
    else
    { // Inside a transaction
      Connection c;
      long tid = request.getTransactionId();
      Long lTid = new Long(tid);

      // Wait for previous writes to complete
      if (waitForCompletionPolicy.getPolicy() != WaitForCompletionPolicy.ALL)
        waitForAllWritesToComplete(backend, request.getTransactionId());

      if (!backend.isStartedTransaction(lTid))
      { // transaction has not been started yet on this backend
        try
        {
          c = cm.getConnection(tid);
        }
        catch (UnreachableBackendException e1)
        {
          logger.error(Translate.get(
              "loadbalancer.backend.disabling.unreachable", backend.getName()));
          backend.disable();
          throw new UnreachableBackendException(Translate.get(
              "loadbalancer.backend.unreacheable", backend.getName()));
        }

        // Sanity check
        if (c == null)
          throw new SQLException(Translate.get(
              "loadbalancer.unable.get.connection", new String[]{
                  String.valueOf(tid), backend.getName()}));

        // begin transaction
        backend.startTransaction(lTid);
        c.setAutoCommit(false);
      }
      else
      { // Re-use the connection used by this transaction
        c = cm.retrieveConnection(tid);

        // Sanity check
        if (c == null)
          throw new SQLException(Translate.get(
              "loadbalancer.unable.retrieve.connection", new String[]{
                  String.valueOf(tid), backend.getName()}));
      }

      // Execute Query
      ControllerResultSet rs = null;
      try
      {
        rs = executeSelectRequestOnBackend(request, backend, c, metadataCache);
      }
      catch (SQLException e)
      {
        throw new SQLException(Translate.get(
            "loadbalancer.request.failed.on.backend", new String[]{
                request.getSQLShortForm(vdb.getSQLShortFormLength()),
                backend.getName(), e.getMessage()}));
      }
      catch (BadConnectionException e)
      { // Connection failed, so did the transaction
        // Disable the backend.
        cm.deleteConnection(tid);
        String msg = Translate.get(
            "loadbalancer.backend.disabling.connection.failure", backend
                .getName());
        logger.error(msg);
        backend.disable();
        throw new SQLException(msg);
      }
      if (logger.isDebugEnabled())
        logger.debug(Translate.get("loadbalancer.execute.transaction.on",
            new String[]{String.valueOf(tid), String.valueOf(request.getId()),
                backend.getName()}));
      return rs;
    }
  }

  protected ControllerResultSet executeStoredProcedureOnBackend(
      StoredProcedure proc, DatabaseBackend backend, MetadataCache metadataCache)
      throws SQLException, UnreachableBackendException
  {
    // Handle macros
    handleMacros(proc);

    // Ok, we have a backend, let's execute the request
    AbstractConnectionManager cm = backend
        .getConnectionManager(proc.getLogin());

    // Sanity check
    if (cm == null)
    {
      String msg = Translate.get("loadbalancer.connectionmanager.not.found",
          new String[]{proc.getLogin(), backend.getName()});
      logger.error(msg);
      throw new SQLException(msg);
    }

    // Execute the query
    if (proc.isAutoCommit())
    {
      if (waitForCompletionPolicy.getPolicy() != WaitForCompletionPolicy.ALL)
        // We could do something finer grain here by waiting
        // only for writes that depend on the tables we need
        // but is that really worth the overhead ?
        waitForAllWritesToComplete(backend);

      // Use a connection just for this request
      Connection c = null;
      try
      {
        c = cm.getConnection();
      }
      catch (UnreachableBackendException e1)
      {
        logger.error(Translate.get(
            "loadbalancer.backend.disabling.unreachable", backend.getName()));
        backend.disable();
        throw new UnreachableBackendException(Translate.get(
            "loadbalancer.backend.unreacheable", backend.getName()));
      }

      // Sanity check
      if (c == null)
        throw new SQLException(Translate.get(
            "loadbalancer.backend.no.connection", backend.getName()));

      // Execute Query
      ControllerResultSet rs = null;
      try
      {
        rs = AbstractLoadBalancer.executeReadStoredProcedureOnBackend(proc,
            backend, c, metadataCache);
      }
      catch (Exception e)
      {
        throw new SQLException(Translate.get(
            "loadbalancer.storedprocedure.failed.on.backend", new String[]{
                proc.getSQLShortForm(vdb.getSQLShortFormLength()),
                backend.getName(), e.getMessage()}));
      }
      finally
      {
        cm.releaseConnection(c);
      }
      if (logger.isDebugEnabled())
        logger.debug(Translate.get("loadbalancer.storedprocedure.on",
            new String[]{String.valueOf(proc.getId()), backend.getName()}));
      return rs;
    }
    else
    { // Inside a transaction
      Connection c;
      long tid = proc.getTransactionId();
      Long lTid = new Long(tid);

      // Wait for previous writes to complete
      if (waitForCompletionPolicy.getPolicy() != WaitForCompletionPolicy.ALL)
        waitForAllWritesToComplete(backend, proc.getTransactionId());

      if (!backend.isStartedTransaction(lTid))
      { // transaction has not been started yet on this backend
        try
        {
          c = cm.getConnection(tid);
        }
        catch (UnreachableBackendException e1)
        {
          logger.error(Translate.get(
              "loadbalancer.backend.disabling.unreachable", backend.getName()));
          backend.disable();
          throw new SQLException(Translate.get(
              "loadbalancer.backend.unreacheable", backend.getName()));
        }

        // Sanity check
        if (c == null)
          throw new SQLException(Translate.get(
              "loadbalancer.unable.get.connection", new String[]{
                  String.valueOf(tid), backend.getName()}));

        // begin transaction
        backend.startTransaction(lTid);
        c.setAutoCommit(false);
      }
      else
      { // Re-use the connection used by this transaction
        c = cm.retrieveConnection(tid);

        // Sanity check
        if (c == null)
          throw new SQLException(Translate.get(
              "loadbalancer.unable.retrieve.connection", new String[]{
                  String.valueOf(tid), backend.getName()}));
      }

      // Execute Query
      ControllerResultSet rs;
      try
      {
        rs = AbstractLoadBalancer.executeReadStoredProcedureOnBackend(proc,
            backend, c, metadataCache);
      }
      catch (Exception e)
      {
        throw new SQLException(Translate.get(
            "loadbalancer.storedprocedure.failed.on.backend", new String[]{
                proc.getSQLShortForm(vdb.getSQLShortFormLength()),
                backend.getName(), e.getMessage()}));
      }
      if (logger.isDebugEnabled())
        logger.debug(Translate.get("loadbalancer.execute.transaction.on",
            new String[]{String.valueOf(tid), String.valueOf(proc.getId()),
                backend.getName()}));
      return rs;
    }
  }

  public ControllerResultSet execReadStoredProcedure(StoredProcedure proc,
      MetadataCache metadataCache) throws SQLException
  {
    ReadStoredProcedureTask task = (ReadStoredProcedureTask) callStoredProcedure(
        proc, true, metadataCache);
    return task.getResult();
  }

  public int execWriteStoredProcedure(StoredProcedure proc) throws SQLException
  {
    WriteStoredProcedureTask task = (WriteStoredProcedureTask) callStoredProcedure(
        proc, false, null);
    return task.getResult();
  }

  private AbstractTask callStoredProcedure(StoredProcedure proc,
      boolean isRead, MetadataCache metadataCache) throws SQLException
  {
    ArrayList backendThreads = backendBlockingThreads;
    ReadPrioritaryFIFOWriteLock lock = backendBlockingThreadsRWLock;

    try
    {
      lock.acquireRead();
    }
    catch (InterruptedException e)
    {
      String msg = Translate.get(
          "loadbalancer.backendlist.acquire.readlock.failed", e);
      logger.error(msg);
      throw new SQLException(msg);
    }

    int nbOfThreads = backendThreads.size();

    // Create the task
    AbstractTask task;
    if (isRead)
      task = new ReadStoredProcedureTask(getNbToWait(nbOfThreads), nbOfThreads,
          proc, metadataCache);
    else
      task = new WriteStoredProcedureTask(getNbToWait(nbOfThreads),
          nbOfThreads, proc);

    synchronized (task)
    {
      int nbOfBackends = 0;

      // Post the task in each backendThread tasklist and wakeup the threads
      for (int i = 0; i < nbOfThreads; i++)
      {
        BackendWorkerThread thread = (BackendWorkerThread) backendThreads
            .get(i);
        if (thread.getBackend().hasStoredProcedure(proc.getProcedureName()))
        {
          nbOfBackends++;
          synchronized (thread)
          {
            if ((waitForCompletionPolicy.getPolicy() == WaitForCompletionPolicy.ALL))
              thread.addTask(task);
            else
              thread.addTask(task, proc.getTransactionId());
            thread.notify();
          }
        }
        if (nbOfBackends == 0)
          throw new SQLException(Translate.get(
              "loadbalancer.backend.no.required.storedprocedure", proc
                  .getProcedureName()));
        else
          task.setTotalNb(nbOfBackends);
      }

      lock.releaseRead();

      // Wait for completion (notified by the task)
      try
      {
        // Wait on task
        long timeout = proc.getTimeout() * 1000;
        if (timeout > 0)
        {
          long start = System.currentTimeMillis();
          task.wait(timeout);
          long end = System.currentTimeMillis();
          long remaining = timeout - (end - start);
          if (remaining <= 0)
          {
            String msg = Translate.get("loadbalancer.storedprocedure.timeout",
                new String[]{String.valueOf(proc.getId()),
                    String.valueOf(task.getSuccess()),
                    String.valueOf(task.getFailed())});

            // Try to remove the request from the task list
            lock.acquireRead();
            nbOfThreads = backendThreads.size();
            for (int i = 0; i < nbOfThreads; i++)
            {
              BackendWorkerThread thread = (BackendWorkerThread) backendThreads
                  .get(i);
              synchronized (thread)
              {
                thread.removeTask(task);
              }
            }
            lock.releaseRead();

            logger.warn(msg);
            throw new SQLException(msg);
          }
          // No need to update request timeout since the execution is finished
        }
        else
          task.wait();
      }
      catch (InterruptedException e)
      {
        // Try to remove the request from the task list
        try
        {
          lock.acquireRead();
          nbOfThreads = backendThreads.size();
        }
        catch (InterruptedException ignore)
        {
          nbOfThreads = 0; // Give up
        }
        for (int i = 0; i < nbOfThreads; i++)
        {
          BackendWorkerThread thread = (BackendWorkerThread) backendThreads
              .get(i);
          synchronized (thread)
          {
            thread.removeTask(task);
          }
        }
        lock.releaseRead();

        throw new SQLException(Translate.get(
            "loadbalancer.storedprocedure.timeout", new String[]{
                String.valueOf(proc.getId()),
                String.valueOf(task.getSuccess()),
                String.valueOf(task.getFailed())}));
      }

      if (task.getSuccess() > 0)
        return task;
      else
      { // All tasks failed
        ArrayList exceptions = task.getExceptions();
        if (exceptions == null)
          throw new SQLException(Translate.get(
              "loadbalancer.storedprocedure.all.failed", proc.getId()));
        else
        {
          String errorMsg = Translate.get(
              "loadbalancer.storedprocedure.failed.stack", proc.getId())
              + "\n";
          for (int i = 0; i < exceptions.size(); i++)
            errorMsg += ((SQLException) exceptions.get(i)).getMessage() + "\n";
          logger.error(errorMsg);
          throw new SQLException(errorMsg);
        }
      }
    }
  }

  /*
   * Transaction management
   */

  public final void begin(TransactionMarkerMetaData tm) throws SQLException
  {
  }

  public void commit(TransactionMarkerMetaData tm) throws SQLException
  {
    if (waitForCompletionPolicy.getPolicy() != WaitForCompletionPolicy.ALL)
      waitForAllWritesToComplete(tm.getTransactionId());

    try
    {
      backendNonBlockingThreadsRWLock.acquireRead();
    }
    catch (InterruptedException e)
    {
      String msg = Translate.get(
          "loadbalancer.backendlist.acquire.readlock.failed", e);
      logger.error(msg);
      throw new SQLException(msg);
    }

    int nbOfThreads = backendNonBlockingThreads.size();
    ArrayList commitList = new ArrayList();
    Long iTid = new Long(tm.getTransactionId());

    // Build the list of backend that need to commit this transaction
    for (int i = 0; i < nbOfThreads; i++)
    {
      BackendWorkerThread thread = (BackendWorkerThread) backendNonBlockingThreads
          .get(i);
      if (thread.getBackend().isStartedTransaction(iTid))
        commitList.add(thread);
    }

    nbOfThreads = commitList.size();
    if (nbOfThreads == 0)
    {
      backendNonBlockingThreadsRWLock.releaseRead();
      return;
    }

    // Create the task
    CommitTask task = new CommitTask(getNbToWait(nbOfThreads), nbOfThreads, tm
        .getTimeout(), tm.getLogin(), tm.getTransactionId());

    synchronized (task)
    {
      // Post the task in each backendThread tasklist and wakeup the threads
      for (int i = 0; i < nbOfThreads; i++)
      {
        BackendWorkerThread thread = (BackendWorkerThread) commitList.get(i);
        synchronized (thread)
        {
          thread.addTask(task);
          thread.notify();
        }
      }

      backendNonBlockingThreadsRWLock.releaseRead();

      // Wait for completion (notified by the task)
      try
      {
        // Wait on task
        long timeout = tm.getTimeout();
        if (timeout > 0)
        {
          long start = System.currentTimeMillis();
          task.wait(timeout);
          long end = System.currentTimeMillis();
          long remaining = timeout - (end - start);
          if (remaining <= 0)
          {
            String msg = Translate.get("loadbalancer.commit.timeout",
                new String[]{String.valueOf(tm.getTransactionId()),
                    String.valueOf(task.getSuccess()),
                    String.valueOf(task.getFailed())});
            logger.warn(msg);
            throw new SQLException(msg);
          }
        }
        else
          task.wait();
      }
      catch (InterruptedException e)
      {
        throw new SQLException(Translate.get("loadbalancer.commit.timeout",
            new String[]{String.valueOf(tm.getTransactionId()),
                String.valueOf(task.getSuccess()),
                String.valueOf(task.getFailed())}));
      }

      if (task.getSuccess() > 0)
        return;
      else
      { // All tasks failed
        ArrayList exceptions = task.getExceptions();
        if (exceptions == null)
          throw new SQLException(Translate.get(
              "loadbalancer.commit.all.failed", tm.getTransactionId()));
        else
        {
          String errorMsg = Translate.get("loadbalancer.commit.failed.stack",
              tm.getTransactionId())
              + "\n";
          for (int i = 0; i < exceptions.size(); i++)
            errorMsg += ((SQLException) exceptions.get(i)).getMessage() + "\n";
          logger.error(errorMsg);
          throw new SQLException(errorMsg);
        }
      }
    }
  }

  public void rollback(TransactionMarkerMetaData tm) throws SQLException
  {
    if (waitForCompletionPolicy.getPolicy() != WaitForCompletionPolicy.ALL)
      waitForAllWritesToComplete(tm.getTransactionId());

    try
    {
      backendNonBlockingThreadsRWLock.acquireRead();
    }
    catch (InterruptedException e)
    {
      String msg = Translate.get(
          "loadbalancer.backendlist.acquire.readlock.failed", e);
      logger.error(msg);
      throw new SQLException(msg);
    }
    int nbOfThreads = backendNonBlockingThreads.size();
    ArrayList rollbackList = new ArrayList();
    Long iTid = new Long(tm.getTransactionId());

    // Build the list of backend that need to rollback this transaction
    for (int i = 0; i < nbOfThreads; i++)
    {
      BackendWorkerThread thread = (BackendWorkerThread) backendNonBlockingThreads
          .get(i);
      if (thread.getBackend().isStartedTransaction(iTid))
        rollbackList.add(thread);
    }

    nbOfThreads = rollbackList.size();
    if (nbOfThreads == 0)
    {
      backendNonBlockingThreadsRWLock.releaseRead();
      return;
    }

    // Create the task
    RollbackTask task = new RollbackTask(getNbToWait(nbOfThreads), nbOfThreads,
        tm.getTimeout(), tm.getLogin(), tm.getTransactionId());

    synchronized (task)
    {
      // Post the task in each backendThread tasklist and wakeup the threads
      for (int i = 0; i < nbOfThreads; i++)
      {
        BackendWorkerThread thread = (BackendWorkerThread) rollbackList.get(i);
        synchronized (thread)
        {
          thread.addTask(task);
          thread.notify();
        }
      }

      backendNonBlockingThreadsRWLock.releaseRead();

      // Wait for completion (notified by the task)
      try
      {
        // Wait on task
        long timeout = tm.getTimeout();
        if (timeout > 0)
        {
          long start = System.currentTimeMillis();
          task.wait(timeout);
          long end = System.currentTimeMillis();
          long remaining = timeout - (end - start);
          if (remaining <= 0)
          {
            String msg = Translate.get("loadbalancer.rollback.timeout",
                new String[]{String.valueOf(tm.getTransactionId()),
                    String.valueOf(task.getSuccess()),
                    String.valueOf(task.getFailed())});
            logger.warn(msg);
            throw new SQLException(msg);
          }
        }
        else
          task.wait();
      }
      catch (InterruptedException e)
      {
        throw new SQLException(Translate.get("loadbalancer.rollback.timeout",
            new String[]{String.valueOf(tm.getTransactionId()),
                String.valueOf(task.getSuccess()),
                String.valueOf(task.getFailed())}));
      }

      if (task.getSuccess() > 0)
        return;
      else
      { // All tasks failed
        ArrayList exceptions = task.getExceptions();
        if (exceptions == null)
          throw new SQLException(Translate.get(
              "loadbalancer.rollback.all.failed", tm.getTransactionId()));
        else
        {
          String errorMsg = Translate.get("loadbalancer.rollback.failed.stack",
              tm.getTransactionId())
              + "\n";
          for (int i = 0; i < exceptions.size(); i++)
            errorMsg += ((SQLException) exceptions.get(i)).getMessage() + "\n";
          logger.error(errorMsg);
          throw new SQLException(errorMsg);
        }
      }
    }
  }

  protected void waitForAllWritesToComplete(long transactionId)
      throws SQLException
  {
    try
    {
      backendBlockingThreadsRWLock.acquireRead();
    }
    catch (InterruptedException e)
    {
      String msg = Translate.get(
          "loadbalancer.backendlist.acquire.readlock.failed", e);
      logger.error(msg);
      throw new SQLException(msg);
    }

    int nbOfThreads = backendBlockingThreads.size();

    for (int i = 0; i < nbOfThreads; i++)
    {
      BackendWorkerThread thread = (BackendWorkerThread) backendBlockingThreads
          .get(i);
      thread.waitForAllTasksToComplete(transactionId);
    }

    backendBlockingThreadsRWLock.releaseRead();
  }

  protected void waitForAllWritesToComplete(DatabaseBackend backend,
      long transactionId) throws SQLException
  {
    try
    {
      backendBlockingThreadsRWLock.acquireRead();
    }
    catch (InterruptedException e)
    {
      String msg = Translate.get(
          "loadbalancer.backendlist.acquire.readlock.failed", e);
      logger.error(msg);
      throw new SQLException(msg);
    }

    int nbOfThreads = backendBlockingThreads.size();

    for (int i = 0; i < nbOfThreads; i++)
    {
      BackendWorkerThread thread = (BackendWorkerThread) backendBlockingThreads
          .get(i);
      if (thread.getBackend() == backend)
        thread.waitForAllTasksToComplete(transactionId);
    }

    backendBlockingThreadsRWLock.releaseRead();
  }

  protected void waitForAllWritesToComplete(DatabaseBackend backend)
      throws SQLException
  {
    try
    {
      backendBlockingThreadsRWLock.acquireRead();
    }
    catch (InterruptedException e)
    {
      String msg = Translate.get(
          "loadbalancer.backendlist.acquire.readlock.failed", e);
      logger.error(msg);
      throw new SQLException(msg);
    }

    int nbOfThreads = backendBlockingThreads.size();

    for (int i = 0; i < nbOfThreads; i++)
    {
      BackendWorkerThread thread = (BackendWorkerThread) backendBlockingThreads
          .get(i);
      if (thread.getBackend() == backend)
        thread.waitForAllTasksToComplete();
    }

    backendBlockingThreadsRWLock.releaseRead();
  }

  /*
   * Backends management
   */

  public void enableBackend(DatabaseBackend db, boolean writeEnabled)
      throws SQLException
  {
    if (writeEnabled)
    {
      // Create 2 worker threads
      BackendWorkerThread blockingThread = new BackendWorkerThread(db, this);
      BackendWorkerThread nonBlockingThread = new BackendWorkerThread(db, this);

      // Add first to the blocking thread list
      try
      {
        backendBlockingThreadsRWLock.acquireWrite();
      }
      catch (InterruptedException e)
      {
        String msg = Translate.get(
            "loadbalancer.backendlist.acquire.writelock.failed", e);
        logger.error(msg);
        throw new SQLException(msg);
      }
      backendBlockingThreads.add(blockingThread);
      backendBlockingThreadsRWLock.releaseWrite();
      blockingThread.start();
      logger.info(Translate.get(
          "loadbalancer.backend.workerthread.blocking.add", db.getName()));

      // Then add to the non-blocking thread list
      try
      {
        backendNonBlockingThreadsRWLock.acquireWrite();
      }
      catch (InterruptedException e)
      {
        String msg = Translate.get(
            "loadbalancer.backendlist.acquire.writelock.failed", e);
        logger.error(msg);
        throw new SQLException(msg);
      }
      backendNonBlockingThreads.add(nonBlockingThread);
      backendNonBlockingThreadsRWLock.releaseWrite();
      nonBlockingThread.start();
      logger.info(Translate.get(
          "loadbalancer.backend.workerthread.non.blocking.add", db.getName()));
      db.enableWrite();
    }

    if (!db.isInitialized())
      db.initializeConnections();
    db.enableRead();
  }

  public synchronized void disableBackend(DatabaseBackend db)
      throws SQLException
  {
    if (db.isWriteEnabled())
    {
      // Start with the backendBlockingThread list
      try
      {
        backendBlockingThreadsRWLock.acquireWrite();
      }
      catch (InterruptedException e)
      {
        String msg = Translate.get(
            "loadbalancer.backendlist.acquire.writelock.failed", e);
        logger.error(msg);
        throw new SQLException(msg);
      }

      int nbOfThreads = backendBlockingThreads.size();

      // Find the right blocking thread
      for (int i = 0; i < nbOfThreads; i++)
      {
        BackendWorkerThread thread = (BackendWorkerThread) backendBlockingThreads
            .get(i);
        if (thread.getBackend().equals(db))
        {
          logger.info(Translate
              .get("loadbalancer.backend.workerthread.blocking.remove", db
                  .getName()));

          backendBlockingThreads.remove(thread);

          synchronized (thread)
          {
            // Kill the thread
            thread.addPriorityTask(new KillThreadTask(1, 1));
            thread.notify();
          }
          break;
        }
      }

      backendBlockingThreadsRWLock.releaseWrite();

      // Continue with the backendNonBlockingThread list

      try
      {
        backendNonBlockingThreadsRWLock.acquireWrite();
      }
      catch (InterruptedException e)
      {
        String msg = Translate.get(
            "loadbalancer.backendlist.acquire.writelock.failed", e);
        logger.error(msg);
        throw new SQLException(msg);
      }

      // Find the right non-blocking thread
      nbOfThreads = backendNonBlockingThreads.size();
      for (int i = 0; i < nbOfThreads; i++)
      {
        BackendWorkerThread thread = (BackendWorkerThread) backendNonBlockingThreads
            .get(i);
        if (thread.getBackend().equals(db))
        {
          logger.info(Translate.get(
              "loadbalancer.backend.workerthread.non.blocking.remove", db
                  .getName()));

          backendNonBlockingThreads.remove(thread);

          synchronized (thread)
          {
            // Kill the thread
            thread.addPriorityTask(new KillThreadTask(1, 1));
            thread.notify();
          }
          break;
        }
      }

      backendNonBlockingThreadsRWLock.releaseWrite();
    }

    db.disable();
    if (db.isInitialized())
      db.finalizeConnections();
  }

  public String getXmlImpl()
  {
    StringBuffer info = new StringBuffer();
    info.append("<" + DatabasesXmlTags.ELT_RAIDb_2 + ">");
    if (createTablePolicy != null)
      info.append(createTablePolicy.getXml());
    if (waitForCompletionPolicy != null)
      info.append(waitForCompletionPolicy.getXml());
    if (macroHandler != null)
      info.append(macroHandler.getXml());
    this.getRaidb2Xml();
    info.append("</" + DatabasesXmlTags.ELT_RAIDb_2 + ">");
    return info.toString();
  }

  public abstract String getRaidb2Xml();

}

---