Developing a client-based JPA loader with a DataGrid agent

When loading from the client side, using a DataGrid agent could increase performance. By using a DataGrid agent, all the data reads and writes occur in the server process. You can also design your application to make sure that DataGrid agents on multiple partitions run in parallel to further boost performance.

About this task

For more information about the DataGrid agent, see DataGrid APIs and partitioning.

After you create the data preload implementation, you can create a generic Loader to complete the following tasks:

Query the data from database in batches.
Build a key list and value list for each partition.
For each partition, call the agentMgr.callReduceAgent(agent, aKey) method to run the agent in the server in a thread. By running in a thread, you can run agents concurrently on multiple partitions.

Example

The following snippet of code is an example of how to load using a DataGrid agent:

import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;

import com.ibm.websphere.objectgrid.NoActiveTransactionException;
import com.ibm.websphere.objectgrid.ObjectGridException;
import com.ibm.websphere.objectgrid.ObjectGridRuntimeException;
import com.ibm.websphere.objectgrid.ObjectMap;
import com.ibm.websphere.objectgrid.Session;
import com.ibm.websphere.objectgrid.TransactionException;
import com.ibm.websphere.objectgrid.datagrid.ReduceGridAgent;
import com.ibm.websphere.objectgrid.em.EntityManager;

public class InsertAgent implements ReduceGridAgent, Externalizable {

    private static final long serialVersionUID = 6568906743945108310L;

    private List keys = null;

    private List vals = null;

    protected boolean isEntityMap;

    public InsertAgent() {
    }

    public InsertAgent(boolean entityMap) {
        isEntityMap = entityMap;
    }

    public Object reduce(Session sess, ObjectMap map) {
        throw new UnsupportedOperationException(
            "ReduceGridAgent.reduce(Session, ObjectMap)");
    }

    public Object reduce(Session sess, ObjectMap map, Collection arg2) {
        Session s = null;
        try {
            s = sess.getObjectGrid().getSession();
            ObjectMap m = s.getMap(map.getName());
            s.beginNoWriteThrough();
            Object ret = process(s, m);
            s.commit();
            return ret;
        } catch (ObjectGridRuntimeException e) {
            if (s.isTransactionActive()) {
                try {
                    s.rollback();
                } catch (TransactionException e1) {
                } catch (NoActiveTransactionException e1) {
                }
            }
            throw e;
        } catch (Throwable t) {
            if (s.isTransactionActive()) {
                try {
                    s.rollback();
                } catch (TransactionException e1) {
                } catch (NoActiveTransactionException e1) {
                }
            }
            throw new ObjectGridRuntimeException(t);
        }

    }

    public Object process(Session s, ObjectMap m) {
        try {

            if (!isEntityMap) {
                // In the POJO case, it is very straightforward, 
                // we can just put everything in the
                // map using insert
                insert(m);
            } else {
                // 2. Entity case.
                // In the Entity case, we can persist the entities
                EntityManager em = s.getEntityManager();
                persistEntities(em);

            }
            return Boolean.TRUE;
        } catch (ObjectGridRuntimeException e) {
            throw e;
        } catch (ObjectGridException e) {
            throw new ObjectGridRuntimeException(e);
        } catch (Throwable t) {
            throw new ObjectGridRuntimeException(t);
        }

    }

    /**
     * Basically this is fresh load.
     * @param s
     * @param m
     * @throws ObjectGridException
     */
    protected void insert(ObjectMap m) throws ObjectGridException {

        int size = keys.size();

        for (int i = 0; i < size; i++) {
            m.insert(keys.get(i), vals.get(i));
        }

    }


    protected void persistEntities(EntityManager em) {
        Iterator<Object> iter = vals.iterator();

        while (iter.hasNext()) {
            Object value = iter.next();
            em.persist(value);
        }
    }

    public Object reduceResults(Collection arg0) {
        return arg0;
    }

    public void readExternal(ObjectInput in) 
      throws IOException, ClassNotFoundException {
        int v = in.readByte();
        isEntityMap = in.readBoolean();
        vals = readList(in);
        if (!isEntityMap) {
            keys = readList(in);
        }
    }

    public void writeExternal(ObjectOutput out) throws IOException {
        out.write(1);
        out.writeBoolean(isEntityMap);

        writeList(out, vals);
        if (!isEntityMap) {
            writeList(out, keys);
        }

    }

    public void setData(List ks, List vs) {
        vals = vs;
        if (!isEntityMap) {
            keys = ks;
        }
    }

    /**
     * @return Returns the isEntityMap.
     */
    public boolean isEntityMap() {
        return isEntityMap;
    }

    static public void writeList(ObjectOutput oo, Collection l) 
      throws IOException {
        int size = l == null ? -1 : l.size();
        oo.writeInt(size);
        if (size > 0) {
            Iterator iter = l.iterator();
            while (iter.hasNext()) {
                Object o = iter.next();
                oo.writeObject(o);
            }
        }
    }

    public static List readList(ObjectInput oi) 
      throws IOException, ClassNotFoundException {
        int size = oi.readInt();
        if (size == -1) {
            return null;
        }


        ArrayList list = new ArrayList(size);
        for (int i = 0; i < size; ++i) {
            Object o = oi.readObject();
            list.add(o);
        }
        return list;
    }
}