Index: src/test/testMapFileB/b1/index
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: src/test/testMapFileB/b1/index
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Index: src/test/testMapFileB/b1/data
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: src/test/testMapFileB/b1/data
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Index: src/test/testMapFileB/b2/index
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: src/test/testMapFileB/b2/index
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Index: src/test/testMapFileB/b2/data
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: src/test/testMapFileB/b2/data
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Index: src/test/java/net/sf/katta/PerformanceTest.java
===================================================================
--- src/test/java/net/sf/katta/PerformanceTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/test/java/net/sf/katta/PerformanceTest.java	(.../trunk/katta)	(revision 10997)
@@ -19,18 +19,17 @@
 import java.util.List;
 import java.util.Random;
 
-import net.sf.katta.client.Client;
-import net.sf.katta.client.IClient;
+import net.sf.katta.client.ILuceneClient;
+import net.sf.katta.client.LuceneClient;
 import net.sf.katta.node.Hit;
 import net.sf.katta.node.Hits;
+import net.sf.katta.node.LuceneServer;
 import net.sf.katta.node.Query;
 import net.sf.katta.testutil.TestResources;
 import net.sf.katta.util.KattaException;
 import net.sf.katta.zk.ZKClient;
 import net.sf.katta.zk.ZkPathes;
 
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-
 public class PerformanceTest extends AbstractKattaTest {
 
   final int _hitCount = 200000;
@@ -44,18 +43,18 @@
     MasterStartThread masterStartThread = startMaster();
     final ZKClient zkClientMaster = masterStartThread.getZkClient();
 
-    NodeStartThread nodeStartThread1 = startNode();
-    NodeStartThread nodeStartThread2 = startNode();
+    NodeStartThread nodeStartThread1 = startNode(new LuceneServer());
+    NodeStartThread nodeStartThread2 = startNode(new LuceneServer());
     masterStartThread.join();
     nodeStartThread1.join();
     nodeStartThread2.join();
     waitForChilds(zkClientMaster, ZkPathes.NODES, 2);
 
     final Katta katta = new Katta();
-    katta.addIndex("index1", TestResources.INDEX1.getAbsolutePath(), StandardAnalyzer.class.getName(), 1);
-    katta.addIndex("index2", TestResources.INDEX2.getAbsolutePath(), StandardAnalyzer.class.getName(), 1);
+    katta.addIndex("index1", TestResources.INDEX1.getAbsolutePath(), 1);
+    katta.addIndex("index2", TestResources.INDEX2.getAbsolutePath(), 1);
 
-    final IClient client = new Client();
+    final ILuceneClient client = new LuceneClient();
     final Query query = new Query("foo: bar");
     long start = System.currentTimeMillis();
     for (int i = 0; i < 10000; i++) {
Index: src/test/java/net/sf/katta/NodeMasterReconnectTest.java
===================================================================
--- src/test/java/net/sf/katta/NodeMasterReconnectTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/test/java/net/sf/katta/NodeMasterReconnectTest.java	(.../trunk/katta)	(revision 10997)
@@ -18,6 +18,7 @@
 import java.util.concurrent.TimeUnit;
 
 import net.sf.katta.master.Master;
+import net.sf.katta.node.LuceneServer;
 import net.sf.katta.node.Node;
 import net.sf.katta.testutil.Gateway;
 import net.sf.katta.util.ZkConfiguration;
@@ -41,7 +42,7 @@
     final MasterStartThread masterStartThread = startMaster();
     final Master master = masterStartThread.getMaster();
     final ZKClient zkNodeClient = new ZKClient(gatewayConf);
-    final Node node = new Node(zkNodeClient);
+    final Node node = new Node(zkNodeClient, new LuceneServer());
     node.start();
     masterStartThread.join();
     final ZKClient zkMasterClient = masterStartThread.getZkClient();
Index: src/test/java/net/sf/katta/zk/ZKClientTest.java
===================================================================
--- src/test/java/net/sf/katta/zk/ZKClientTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/test/java/net/sf/katta/zk/ZKClientTest.java	(.../trunk/katta)	(revision 10997)
@@ -28,7 +28,6 @@
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher.Event;
 import org.apache.zookeeper.Watcher.Event.EventType;
 import org.apache.zookeeper.Watcher.Event.KeeperState;
 import org.jmock.Expectations;
@@ -112,13 +111,12 @@
     if (client.exists(katta)) {
       client.deleteRecursive(katta);
     }
-    client.create(katta, new IndexMetaData("path", "someAnalyzr", 3, IndexMetaData.IndexState.ANNOUNCED));
+    client.create(katta, new IndexMetaData("path", 3, IndexMetaData.IndexState.ANNOUNCED));
     client.subscribeDataChanges(katta, listener);
     for (int i = 0; i < 10; i++) {
       client.getEventLock().lock();
       try{
-        final IndexMetaData indexMetaData = new IndexMetaData("path", "someAnalyzr" + i, 3,
-            IndexMetaData.IndexState.ANNOUNCED);
+        final IndexMetaData indexMetaData = new IndexMetaData("path", 3, IndexMetaData.IndexState.ANNOUNCED);
         client.writeData(katta, indexMetaData);
         client.getEventLock().getDataChangedCondition().await();
       } finally {
Index: src/test/java/net/sf/katta/node/SleepServerTest.java
===================================================================
--- src/test/java/net/sf/katta/node/SleepServerTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/test/java/net/sf/katta/node/SleepServerTest.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,66 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.node;
+
+import net.sf.katta.testutil.ExtendedTestCase;
+import net.sf.katta.util.SleepServer;
+
+import org.apache.log4j.Logger;
+
+/**
+ * Test for {@link SleepServer}.
+ */
+public class SleepServerTest extends ExtendedTestCase {
+
+  @SuppressWarnings("unused")
+  private static Logger LOG = Logger.getLogger(SleepServerTest.class);
+
+
+  public void testNoSleep() throws Exception {
+    SleepServer server = new SleepServer();
+    long start = System.currentTimeMillis();
+    server.sleep(0, 0);
+    long time = System.currentTimeMillis() - start;
+    assertTrue(time < 10);
+  }
+  
+  public void testSleep() throws Exception {
+    SleepServer server = new SleepServer();
+    long start = System.currentTimeMillis();
+    server.sleep(100, 0);
+    long time = System.currentTimeMillis() - start;
+    assertTrue(time >= 100);
+  }
+  
+  public void testVariation() throws Exception {
+    SleepServer server = new SleepServer();
+    long min = Integer.MAX_VALUE;
+    long max = -1;
+    for (int i=0; i<200; i++) {
+      long n = checkTime(server);
+      max = Math.max(n, max);
+      min = Math.min(n, min);
+    }
+    assertTrue(max - min >= 9);
+  }
+  
+  private long checkTime(SleepServer server) {
+    long start = System.currentTimeMillis();
+    server.sleep(10, 5);
+    return System.currentTimeMillis() - start;
+  }
+
+}
Index: src/test/java/net/sf/katta/node/MapFileServerTest.java
===================================================================
--- src/test/java/net/sf/katta/node/MapFileServerTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/test/java/net/sf/katta/node/MapFileServerTest.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,208 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.node;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import net.sf.katta.testutil.ExtendedTestCase;
+import net.sf.katta.testutil.TestResources;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.log4j.Logger;
+
+/**
+ * Test for {@link MapFileServer }.
+ */
+public class MapFileServerTest extends ExtendedTestCase {
+
+  @SuppressWarnings("unused")
+  private static Logger LOG = Logger.getLogger(MapFileServerTest.class);
+
+  private static final String NODE_NAME = "TestNode";
+  private static final String SHARD_A_1 = "shard_A_1";
+  private static final String SHARD_A_2 = "shard_A_2";
+  private static final String SHARD_A_3 = "shard_A_3";
+  private static final String SHARD_A_4 = "shard_A_4";
+  private static final String SHARD_B_1 = "shard_B_1";
+  private static final String SHARD_B_2 = "shard_B_2";
+
+  public void testShardA1() throws Exception {
+    MapFileServer server = new MapFileServer();
+    server.setNodeName(NODE_NAME);
+    server.addShard(SHARD_A_1, new File(TestResources.MAP_FILE_A, "a1"));
+    assertEquals(3, server.shardSize(SHARD_A_1));
+    String[] shards = new String[] { SHARD_A_1 };
+    assertEquals("This is a test", getOneResult(server, "a.txt", shards));
+    assertMissing(server, "d.html", shards);
+    assertMissing(server, "v.xml", shards);
+    assertMissing(server, "y.xml", shards);
+    assertMissing(server, "not-found", shards);
+    server.shutdown();
+  }
+
+  public void testShardA2() throws Exception {
+    MapFileServer server = new MapFileServer();
+    server.setNodeName(NODE_NAME);
+    server.addShard(SHARD_A_2, new File(TestResources.MAP_FILE_A, "a2"));
+    assertEquals(3, server.shardSize(SHARD_A_2));
+    String[] shards = new String[] { SHARD_A_2 };
+    assertEquals("<b>test</b>", getOneResult(server, "d.html", shards));
+    assertMissing(server, "a.txt", shards);
+    assertMissing(server, "v.xml", shards);
+    assertMissing(server, "y.xml", shards);
+    assertMissing(server, "not-found", shards);
+    server.shutdown();
+  }
+
+  public void testMapFile1() throws Exception {
+    MapFileServer server = new MapFileServer();
+    server.setNodeName(NODE_NAME);
+    server.addShard(SHARD_A_1, new File(TestResources.MAP_FILE_A, "a1"));
+    server.addShard(SHARD_A_2, new File(TestResources.MAP_FILE_A, "a2"));
+    server.addShard(SHARD_A_3, new File(TestResources.MAP_FILE_A, "a3"));
+    server.addShard(SHARD_A_4, new File(TestResources.MAP_FILE_A, "a4"));
+    assertEquals(3, server.shardSize(SHARD_A_1));
+    assertEquals(3, server.shardSize(SHARD_A_2));
+    assertEquals(2, server.shardSize(SHARD_A_3));
+    assertEquals(4, server.shardSize(SHARD_A_4));
+    String[] shards = new String[] { SHARD_A_1, SHARD_A_2, SHARD_A_3, SHARD_A_4 };
+    assertEquals("This is a test", getOneResult(server, "a.txt", shards));
+    assertEquals("<b>test</b>", getOneResult(server, "d.html", shards));
+    assertEquals("Test in part 3", getOneResult(server, "h.txt", shards));
+    assertEquals("test data", getOneResult(server, "k.out", shards));
+    assertMissing(server, "v.xml", shards);
+    assertMissing(server, "y.xml", shards);
+    assertMissing(server, "not-found", shards);
+   server.shutdown();
+  }
+  
+  public void testBothMapFiles() throws Exception {
+    MapFileServer server = new MapFileServer();
+    server.setNodeName(NODE_NAME);
+    server.addShard(SHARD_A_1, new File(TestResources.MAP_FILE_A, "a1"));
+    server.addShard(SHARD_A_2, new File(TestResources.MAP_FILE_A, "a2"));
+    server.addShard(SHARD_A_3, new File(TestResources.MAP_FILE_A, "a3"));
+    server.addShard(SHARD_A_4, new File(TestResources.MAP_FILE_A, "a4"));
+    server.addShard(SHARD_B_1, new File(TestResources.MAP_FILE_B, "b1"));
+    server.addShard(SHARD_B_2, new File(TestResources.MAP_FILE_B, "b2"));
+    assertEquals(3, server.shardSize(SHARD_A_1));
+    assertEquals(3, server.shardSize(SHARD_A_2));
+    assertEquals(2, server.shardSize(SHARD_A_3));
+    assertEquals(4, server.shardSize(SHARD_A_4));
+    assertEquals(3, server.shardSize(SHARD_B_1));
+    assertEquals(3, server.shardSize(SHARD_B_2));
+    String[] shards = new String[] { SHARD_A_1, SHARD_A_2, SHARD_A_3, SHARD_A_4, SHARD_B_1, SHARD_B_2 };
+    String[] mf1Shards = new String[] { SHARD_A_1, SHARD_A_2, SHARD_A_3, SHARD_A_4 };
+    String[] mf2Shards = new String[] { SHARD_B_1, SHARD_B_2 };
+    assertEquals("This is a test", getOneResult(server, "a.txt", shards));
+    assertMissing(server, "a.txt", mf2Shards);
+    assertEquals("<b>test</b>", getOneResult(server, "d.html", shards));
+    assertMissing(server, "d.html", mf2Shards);
+    assertEquals("Test in part 3", getOneResult(server, "h.txt", shards));
+    assertMissing(server, "h.txt", mf2Shards);
+    assertEquals("test data", getOneResult(server, "k.out", shards));
+    assertMissing(server, "k.out", mf2Shards);
+    assertEquals("where is test", getOneResult(server, "w.txt", shards));
+    assertMissing(server, "w.txt", mf1Shards);
+    assertEquals("xrays ionize", getOneResult(server, "x.txt", shards));
+    assertMissing(server, "x.txt", mf1Shards);
+    assertMissing(server, "not-found", shards);
+    server.shutdown();
+  }
+  
+  public void testMultiThreadedAccess() throws Exception {
+    final MapFileServer server = new MapFileServer();
+    server.setNodeName(NODE_NAME);
+    server.addShard(SHARD_A_1, new File(TestResources.MAP_FILE_A, "a1"));
+    server.addShard(SHARD_A_2, new File(TestResources.MAP_FILE_A, "a2"));
+    server.addShard(SHARD_A_3, new File(TestResources.MAP_FILE_A, "a3"));
+    server.addShard(SHARD_A_4, new File(TestResources.MAP_FILE_A, "a4"));
+    server.addShard(SHARD_B_1, new File(TestResources.MAP_FILE_B, "b1"));
+    server.addShard(SHARD_B_2, new File(TestResources.MAP_FILE_B, "b2"));
+    final String[] shards = new String[] { SHARD_A_1, SHARD_A_2, SHARD_A_3, SHARD_A_4, SHARD_B_1, SHARD_B_2 };
+    final Map<String, String> entries = new HashMap<String, String>();
+    entries.put("a.txt", "This is a test");
+    entries.put("b.xml", "<name>test</name>");
+    entries.put("d.html", "<b>test</b>");
+    entries.put("h.txt", "Test in part 3");
+    entries.put("i.xml", "<i>test</i>");
+    entries.put("k.out", "test data");
+    entries.put("w.txt", "where is test");
+    entries.put("x.txt", "xrays ionize");
+    entries.put("z.xml", "<zed>foo</zed>");
+    final List<String> keys = new ArrayList<String>(entries.keySet());
+    Random rand = new Random("katta".hashCode());
+    List<Thread> threads = new ArrayList<Thread>();
+    final List<Exception> exceptions = new ArrayList<Exception>();
+    long startTime = System.currentTimeMillis();
+    final AtomicInteger count = new AtomicInteger(0);
+    for (int i=0; i<20; i++) {
+      final Random rand2 = new Random(rand.nextInt());
+      Thread t = new Thread(new Runnable() {
+        public void run() {
+          for (int j=0; j<500; j++) {
+            int n = rand2.nextInt(entries.size());
+            String key = keys.get(n);
+            try {
+              assertEquals(entries.get(key), getOneResult(server, key, shards));
+              count.incrementAndGet();
+            } catch (Exception e) {
+              System.err.println(e);
+              exceptions.add(e);
+              break;
+            }
+          }
+        }
+      });
+      threads.add(t);
+      t.start();
+    }
+    for (Thread t : threads) {
+      t.join();
+    }
+    long time = System.currentTimeMillis() - startTime;
+    System.out.println((1000.0 * (double) count.intValue() / (double) time) + " requests / sec");
+    assertTrue(exceptions.isEmpty());
+  }
+  
+  
+  private String getOneResult(IMapFileServer server, String key, String[] shards) throws Exception {
+    TextArrayWritable texts = server.get(new Text(key), shards);
+    assertNotNull(texts);
+    assertNotNull(texts.array);
+    Writable[] array = texts.array.get();
+    assertEquals(1, array.length);
+    assertTrue(array[0] instanceof Text);
+    Text text = (Text) array[0];
+    return text.toString();
+  }
+
+  private void assertMissing(IMapFileServer server, String key, String[] shards) throws Exception {
+    TextArrayWritable texts = server.get(new Text(key), shards);
+    assertNotNull(texts);
+    assertNotNull(texts.array);
+    Writable[] array = texts.array.get();
+    assertEquals(0, array.length);
+ }
+  
+}
Index: src/test/java/net/sf/katta/node/NodeTest.java
===================================================================
--- src/test/java/net/sf/katta/node/NodeTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/test/java/net/sf/katta/node/NodeTest.java	(.../trunk/katta)	(revision 10997)
@@ -24,7 +24,6 @@
 import java.util.concurrent.Future;
 
 import junit.framework.Assert;
-
 import net.sf.katta.AbstractKattaTest;
 import net.sf.katta.Katta;
 import net.sf.katta.index.AssignedShard;
@@ -35,7 +34,6 @@
 import net.sf.katta.zk.ZkPathes;
 
 import org.apache.lucene.analysis.KeywordAnalyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.queryParser.QueryParser;
 import org.apache.lucene.search.Query;
 import org.mockito.Mockito;
@@ -44,14 +42,14 @@
 
   public void testShardStatusSuccess() throws Exception {
     MasterStartThread masterThread = startMaster();
-    NodeStartThread nodeThread = startNode();
+    NodeStartThread nodeThread = startNode(new LuceneServer());
     masterThread.join();
     nodeThread.join();
     waitForChilds(masterThread.getZkClient(), ZkPathes.NODES, 1);
 
     // deploy index
     Katta katta = new Katta();
-    katta.addIndex("index", TestResources.INDEX1.getAbsolutePath(), StandardAnalyzer.class.getName(), 1);
+    katta.addIndex("index", TestResources.INDEX1.getAbsolutePath(), 1);
 
     // test
     final String indexPath = ZkPathes.INDEXES + "/index";
@@ -67,14 +65,14 @@
 
   public void testShardStatusNoSuccessNoIndexGiven() throws Exception {
     MasterStartThread masterThread = startMaster();
-    NodeStartThread nodeThread = startNode();
+    NodeStartThread nodeThread = startNode(new LuceneServer());
     masterThread.join();
     nodeThread.join();
     waitForChilds(masterThread.getZkClient(), ZkPathes.NODES, 1);
 
     // deploy index
     Katta katta = new Katta();
-    katta.addIndex("index", "src/test/testIndexNotHere/", StandardAnalyzer.class.getName(), 1);
+    katta.addIndex("index", "src/test/testIndexNotHere/", 1);
 
     // test
     final String indexPath = ZkPathes.INDEXES + "/index";
@@ -91,7 +89,7 @@
 
   public void testDeployShardAfterRestart() throws Exception {
     MasterStartThread masterThread = startMaster();
-    NodeStartThread nodeThread = startNode();
+    NodeStartThread nodeThread = startNode(new LuceneServer());
     masterThread.join();
     nodeThread.join();
     waitForChilds(masterThread.getZkClient(), ZkPathes.NODES, 1);
@@ -101,7 +99,7 @@
     assertEquals(0, node.getDeployedShards().size());
     Katta katta = new Katta();
     String index = "index";
-    katta.addIndex(index, TestResources.INDEX1.getAbsolutePath(), StandardAnalyzer.class.getName(), 1);
+    katta.addIndex(index, TestResources.INDEX1.getAbsolutePath(), 1);
 
     // test
     assertTrue(node.getDeployedShards().size() > 0);
@@ -110,7 +108,7 @@
     assertEquals(IndexMetaData.IndexState.DEPLOYED, indexMetaData.getState());
 
     nodeThread.shutdown();
-    nodeThread = startNode();
+    nodeThread = startNode(new LuceneServer());
     nodeThread.join();
     node = nodeThread.getNode();
     assertTrue(node.getDeployedShards().size() > 0);
@@ -126,7 +124,8 @@
     ZKClient zkClient = Mockito.mock(ZKClient.class);
     Mockito.when(zkClient.getEventLock()).thenReturn(new ZKClient.ZkLock());
 
-    Node node = new Node(zkClient);
+    LuceneServer server = new LuceneServer();
+    Node node = new Node(zkClient, server);
     node.start();
 
     List<AssignedShard> shards = new ArrayList<AssignedShard>();
@@ -147,12 +146,12 @@
     QueryWritable writable = new QueryWritable(query);
 
     String[] shardArray = shardNames.toArray(new String[shardNames.size()]);
-    DocumentFrequenceWritable freqs = node.getDocFreqs(writable, shardArray);
+    DocumentFrequenceWritable freqs = server.getDocFreqs(writable, shardArray);
 
     ExecutorService es = Executors.newFixedThreadPool(100);
     List<Future<HitsMapWritable>> tasks = new ArrayList<Future<HitsMapWritable>>();
     for (int i = 0; i < 10000; i++) {
-      QueryClient client = new QueryClient(node, freqs, writable, shardArray);
+      QueryClient client = new QueryClient(server, freqs, writable, shardArray);
       Future<HitsMapWritable> future = es.submit(client);
       tasks.add(future);
     }
@@ -174,7 +173,7 @@
     ZKClient zkClient = Mockito.mock(ZKClient.class);
     Mockito.when(zkClient.getEventLock()).thenReturn(new ZKClient.ZkLock());
 
-    Node node = new Node(zkClient);
+    Node node = new Node(zkClient, new LuceneServer());
     node.start();
 
     List<AssignedShard> shards = new ArrayList<AssignedShard>();
@@ -196,13 +195,13 @@
 
   private class QueryClient implements Callable<HitsMapWritable> {
 
-    private Node _node;
+    private LuceneServer _server;
     private QueryWritable _query;
     private DocumentFrequenceWritable _freqs;
     private String[] _shards;
 
-    public QueryClient(Node node, DocumentFrequenceWritable freqs, QueryWritable query, String[] shards) {
-      _node = node;
+    public QueryClient(LuceneServer server, DocumentFrequenceWritable freqs, QueryWritable query, String[] shards) {
+      _server = server;
       _freqs = freqs;
       _query = query;
       _shards = shards;
@@ -210,7 +209,7 @@
 
     @Override
     public HitsMapWritable call() throws Exception {
-      return _node.search(_query, _freqs, _shards, 2);
+      return _server.search(_query, _freqs, _shards, 2);
     }
 
   }
Index: src/test/java/net/sf/katta/AbstractKattaTest.java
===================================================================
--- src/test/java/net/sf/katta/AbstractKattaTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/test/java/net/sf/katta/AbstractKattaTest.java	(.../trunk/katta)	(revision 10997)
@@ -19,6 +19,7 @@
 import java.util.concurrent.TimeUnit;
 
 import net.sf.katta.master.Master;
+import net.sf.katta.node.INodeManaged;
 import net.sf.katta.node.Node;
 import net.sf.katta.testutil.ExtendedTestCase;
 import net.sf.katta.util.FileUtil;
@@ -151,25 +152,25 @@
     return masterStartThread;
   }
 
-  protected NodeStartThread startNode() {
-    return startNode(new NodeConfiguration().getShardFolder().getAbsolutePath());
+  protected NodeStartThread startNode(INodeManaged server) {
+    return startNode(server, new NodeConfiguration().getShardFolder().getAbsolutePath());
   }
 
-  protected NodeStartThread startNode(int port) {
-    return startNode(new NodeConfiguration().getShardFolder().getAbsolutePath(), port);
+  protected NodeStartThread startNode(INodeManaged server, int port) {
+    return startNode(server, port, new NodeConfiguration().getShardFolder().getAbsolutePath());
   }
 
-  protected NodeStartThread startNode(String shardFolder) {
+  protected NodeStartThread startNode(INodeManaged server, String shardFolder) {
     NodeConfiguration nodeConf = new NodeConfiguration();
-    return startNode(shardFolder, nodeConf.getStartPort());
+    return startNode(server, nodeConf.getStartPort(), shardFolder);
   }
 
-  protected NodeStartThread startNode(String shardFolder, int port) {
+  protected NodeStartThread startNode(INodeManaged server, int port, String shardFolder) {
     ZKClient zkNodeClient = new ZKClient(_conf);
     NodeConfiguration nodeConf = new NodeConfiguration();
     nodeConf.setShardFolder(shardFolder);
     nodeConf.setStartPort(port);
-    Node node = new Node(zkNodeClient, nodeConf);
+    Node node = new Node(zkNodeClient, nodeConf, server);
     NodeStartThread nodeStartThread = new NodeStartThread(node, zkNodeClient);
     nodeStartThread.start();
     return nodeStartThread;
Index: src/test/java/net/sf/katta/master/FailTest.java
===================================================================
--- src/test/java/net/sf/katta/master/FailTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/test/java/net/sf/katta/master/FailTest.java	(.../trunk/katta)	(revision 10997)
@@ -15,12 +15,11 @@
  */
 package net.sf.katta.master;
 
-import java.util.concurrent.TimeUnit;
-
 import net.sf.katta.AbstractKattaTest;
-import net.sf.katta.client.Client;
 import net.sf.katta.client.DeployClient;
 import net.sf.katta.client.IDeployClient;
+import net.sf.katta.client.LuceneClient;
+import net.sf.katta.node.LuceneServer;
 import net.sf.katta.node.Node;
 import net.sf.katta.node.Query;
 import net.sf.katta.testutil.TestResources;
@@ -30,8 +29,7 @@
 import net.sf.katta.zk.ZKClient;
 import net.sf.katta.zk.ZkPathes;
 
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-
+@SuppressWarnings("deprecation")
 public class FailTest extends AbstractKattaTest {
 
 
@@ -88,9 +86,8 @@
     // deploy index
     final IDeployClient deployClient = new DeployClient(_conf);
     final String indexName = "index";
-    deployClient.addIndex(indexName, TestResources.UNZIPPED_INDEX.getAbsolutePath(), StandardAnalyzer.class.getName(),
-            3).joinDeployment();
-    final Client client = new Client();
+    deployClient.addIndex(indexName, TestResources.UNZIPPED_INDEX.getAbsolutePath(), 3).joinDeployment();
+    final LuceneClient client = new LuceneClient();
     assertEquals(2, client.count(new Query("foo:bar"), new String[] { indexName }));
     assertEquals(1, node1.countShards());
     assertEquals(1, node2.countShards());
@@ -123,7 +120,7 @@
 
     public DummyNode(final ZkConfiguration conf, final NodeConfiguration nodeConfiguration) throws KattaException {
       _client = new ZKClient(conf);
-      _node = new Node(_client, nodeConfiguration);
+      _node = new Node(_client, nodeConfiguration, new LuceneServer());
       _node.start();
     }
 
Index: src/test/java/net/sf/katta/master/MasterTest.java
===================================================================
--- src/test/java/net/sf/katta/master/MasterTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/test/java/net/sf/katta/master/MasterTest.java	(.../trunk/katta)	(revision 10997)
@@ -24,6 +24,7 @@
 import net.sf.katta.client.IIndexDeployFuture;
 import net.sf.katta.index.IndexMetaData;
 import net.sf.katta.index.IndexMetaData.IndexState;
+import net.sf.katta.node.LuceneServer;
 import net.sf.katta.node.Node;
 import net.sf.katta.node.NodeMetaData;
 import net.sf.katta.node.Node.NodeState;
@@ -32,8 +33,6 @@
 import net.sf.katta.zk.ZKClient;
 import net.sf.katta.zk.ZkPathes;
 
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-
 public class MasterTest extends AbstractKattaTest {
 
   private static final String SECOND_SHARD_FOLDER = "/tmp/katta-shards2";
@@ -112,8 +111,8 @@
     final MasterStartThread masterStartThread = startMaster();
     final ZKClient zkClientMaster = masterStartThread.getZkClient();
 
-    final NodeStartThread nodeStartThread1 = startNode();
-    final NodeStartThread nodeStartThread2 = startNode(SECOND_SHARD_FOLDER);
+    final NodeStartThread nodeStartThread1 = startNode(new LuceneServer());
+    final NodeStartThread nodeStartThread2 = startNode(new LuceneServer(), SECOND_SHARD_FOLDER);
     final Node node1 = nodeStartThread1.getNode();
     final Node node2 = nodeStartThread2.getNode();
     masterStartThread.join();
@@ -126,7 +125,7 @@
     final File indexFile = TestResources.INDEX1;
     final Katta katta = new Katta();
     final String index = "indexA";
-    katta.addIndex(index, "file://" + indexFile.getAbsolutePath(), StandardAnalyzer.class.getName(), 2);
+    katta.addIndex(index, "file://" + indexFile.getAbsolutePath(), 2);
 
     final int shardCount = indexFile.list(FileUtil.VISIBLE_FILES_FILTER).length;
     assertEquals(shardCount, zkClientMaster.countChildren(ZkPathes.getIndexPath(index)));
@@ -163,8 +162,8 @@
     final MasterStartThread masterStartThread = startMaster();
     final ZKClient zkClientMaster = masterStartThread.getZkClient();
 
-    final NodeStartThread nodeStartThread1 = startNode();
-    final NodeStartThread nodeStartThread2 = startNode(SECOND_SHARD_FOLDER);
+    final NodeStartThread nodeStartThread1 = startNode(new LuceneServer());
+    final NodeStartThread nodeStartThread2 = startNode(new LuceneServer(), SECOND_SHARD_FOLDER);
     final Node node1 = nodeStartThread1.getNode();
     final Node node2 = nodeStartThread2.getNode();
     masterStartThread.join();
@@ -176,8 +175,7 @@
     final File indexFile = TestResources.INDEX1;
     DeployClient deployClient = new DeployClient(_conf);
     final String index = "indexA";
-    IIndexDeployFuture deployFuture = deployClient.addIndex(index, "file://" + indexFile.getAbsolutePath(),
-        StandardAnalyzer.class.getName(), 1);
+    IIndexDeployFuture deployFuture = deployClient.addIndex(index, "file://" + indexFile.getAbsolutePath(), 1);
     deployFuture.joinDeployment();
 
     final int shardCount = indexFile.list(FileUtil.VISIBLE_FILES_FILTER).length;
@@ -216,8 +214,8 @@
     final MasterStartThread masterStartThread = startMaster();
     final ZKClient zkClientMaster = masterStartThread.getZkClient();
 
-    final NodeStartThread nodeStartThread1 = startNode();
-    final NodeStartThread nodeStartThread2 = startNode(SECOND_SHARD_FOLDER);
+    final NodeStartThread nodeStartThread1 = startNode(new LuceneServer());
+    final NodeStartThread nodeStartThread2 = startNode(new LuceneServer(), SECOND_SHARD_FOLDER);
     masterStartThread.join();
     nodeStartThread1.join();
     nodeStartThread2.join();
@@ -227,7 +225,7 @@
     final File indexFile = TestResources.INVALID_INDEX;
     final Katta katta = new Katta();
     final String index = "indexA";
-    katta.addIndex(index, "file://" + indexFile.getAbsolutePath(), StandardAnalyzer.class.getName(), 2);
+    katta.addIndex(index, "file://" + indexFile.getAbsolutePath(), 2);
 
     final IndexMetaData metaData = new IndexMetaData();
     zkClientMaster.readData(ZkPathes.getIndexPath(index), metaData);
@@ -242,7 +240,7 @@
     MasterStartThread masterStartThread = startMaster();
     final ZKClient zkClientMaster = masterStartThread.getZkClient();
 
-    final NodeStartThread nodeStartThread = startNode();
+    final NodeStartThread nodeStartThread = startNode(new LuceneServer());
     masterStartThread.join();
     nodeStartThread.join();
     waitForPath(zkClientMaster, ZkPathes.MASTER);
@@ -254,7 +252,7 @@
 
     final Katta katta = new Katta();
     final String index = "indexA";
-    katta.addIndex(index, "file://" + indexFile.getAbsolutePath(), StandardAnalyzer.class.getName(), 2);
+    katta.addIndex(index, "file://" + indexFile.getAbsolutePath(), 2);
     assertEquals(shardCount, zkClientMaster.countChildren(ZkPathes.getIndexPath(index)));
 
     // restartmaster
@@ -275,7 +273,7 @@
     final Master master = masterStartThread.getMaster();
 
     // start one node
-    final NodeStartThread nodeStartThread1 = startNode();
+    final NodeStartThread nodeStartThread1 = startNode(new LuceneServer());
     masterStartThread.join();
     waitOnNodes(masterStartThread, 1);
 
@@ -283,8 +281,7 @@
     final File indexFile = TestResources.INDEX1;
     final String index = "indexA";
     final DeployClient deployClient = new DeployClient(zkClient);
-    final IIndexDeployFuture deployFuture = deployClient.addIndex(index, "file://" + indexFile.getAbsolutePath(),
-        StandardAnalyzer.class.getName(), 2);
+    final IIndexDeployFuture deployFuture = deployClient.addIndex(index, "file://" + indexFile.getAbsolutePath(), 2);
     deployFuture.joinDeployment();
     assertEquals(1, deployClient.getIndexes(IndexState.DEPLOYED).size());
     final List<String> shards = zkClient.getChildren(ZkPathes.getIndexPath(index));
@@ -296,7 +293,7 @@
     zkClientMaster.getEventLock().lock();
     NodeStartThread nodeStartThread2;
     try {
-      nodeStartThread2 = startNode(SECOND_SHARD_FOLDER);
+      nodeStartThread2 = startNode(new LuceneServer(), SECOND_SHARD_FOLDER);
       zkClientMaster.getEventLock().getDataChangedCondition().await();
     } finally {
       zkClientMaster.getEventLock().unlock();
Index: src/test/java/net/sf/katta/testutil/TestResources.java
===================================================================
--- src/test/java/net/sf/katta/testutil/TestResources.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/test/java/net/sf/katta/testutil/TestResources.java	(.../trunk/katta)	(revision 10997)
@@ -27,4 +27,7 @@
 
   public final static File SHARD1 = new File(INDEX2, "aIndex.zip");
 
+  public final static File MAP_FILE_A = new File("src/test/testMapFileA");
+  public final static File MAP_FILE_B = new File("src/test/testMapFileB");
+  
 }
Index: src/test/java/net/sf/katta/util/SleepServer.java
===================================================================
--- src/test/java/net/sf/katta/util/SleepServer.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/test/java/net/sf/katta/util/SleepServer.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,64 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+
+import net.sf.katta.node.INodeManaged;
+
+/**
+ * This class implements the back-end side of a dummy server, to be used
+ * for testing. It just sleeps for a while and then returns nothing.
+ */
+public class SleepServer implements INodeManaged, ISleepServer {
+
+  private Random rand = new Random();
+  
+  public long getProtocolVersion(final String protocol, final long clientVersion) throws IOException {
+    return 0L;
+  }
+
+  public void setNodeName(String nodeName) {
+  }
+
+  public void addShard(final String shardName, final File shardDir) throws IOException {
+  }
+
+  public void removeShard(final String shardName) throws IOException {
+  }
+  
+  public int shardSize(final String shardName) {
+    return 0;
+  }
+  
+  public void shutdown() {
+  }
+  
+  public void sleep(long msec, int delta) {
+    if (delta > 0) {
+      msec = Math.max(0, msec + Math.round(((2.0 * rand.nextDouble()) - 1.0) * delta));
+    }
+    if (msec > 0) {
+      try {
+        Thread.sleep(msec);
+      } catch (InterruptedException e) {
+      }
+    }
+  }
+
+}
Index: src/test/java/net/sf/katta/util/ISleepClient.java
===================================================================
--- src/test/java/net/sf/katta/util/ISleepClient.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/test/java/net/sf/katta/util/ISleepClient.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,52 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.util;
+
+/**
+ * The public interface for the front end of a dummy server. It just sleeps
+ * for a while then returns nothing.
+ */
+public interface ISleepClient {
+
+  /**
+   * Sleep for the given number of milliseconds.
+   * @param msec How long each node should sleep for.
+   * @param shards Which shards to send the request to. Use this to control
+   *     which nodes will sleep. Within a node, the shard list is ignored.
+   *     The call will return after all nodes have finished sleeping.
+   * @throws KattaException If an IO exception occurs.
+   */
+  public void sleep(long msec, String[] shards) throws KattaException;
+  
+  /**
+   * Sleep for the given number of milliseconds, +- a random delta.
+   * @param msec How long each node should sleep for.
+   * @param delta The maximum size of the delta (msec) to add or remove
+   *     from the specified time. The node will choose an evenly distributed
+   *     random sleep time from msec-delta to msec+delta.
+   * @param shards Which shards to send the request to. Use this to control
+   *     which nodes will sleep. Within a node, the shard list is ignored.
+   *     The call will return after all nodes have finished sleeping.
+   * @throws KattaException If an IO exception occurs.
+   */
+  public void sleep(long msec, int delta, String[] shards) throws KattaException;
+
+  /**
+   * Closes down the client. Does nothing.
+   */
+  public void close();
+
+}
\ No newline at end of file
Index: src/test/java/net/sf/katta/util/SleepClient.java
===================================================================
--- src/test/java/net/sf/katta/util/SleepClient.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/test/java/net/sf/katta/util/SleepClient.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,73 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.util;
+
+import java.lang.reflect.Method;
+
+import net.sf.katta.client.Client;
+import net.sf.katta.client.INodeSelectionPolicy;
+
+import org.apache.log4j.Logger;
+
+/**
+ * The front end for a test server that just sleeps for a while then
+ * returns nothing. Used for testing.
+ */
+public class SleepClient implements ISleepClient {
+
+  protected final static Logger LOG = Logger.getLogger(SleepClient.class);
+  
+  private Client kattaClient;
+//  private static MethodCache methodCache = new MethodCache(ILuceneSearch.class);
+  
+  public SleepClient(final INodeSelectionPolicy nodeSelectionPolicy) throws KattaException {
+    kattaClient = new Client(ISleepServer.class, nodeSelectionPolicy);
+  }
+
+  public SleepClient() throws KattaException {
+    kattaClient = new Client(ISleepServer.class);
+  }
+
+  public SleepClient(final INodeSelectionPolicy policy, final ZkConfiguration config) throws KattaException {
+    kattaClient = new Client(ISleepServer.class, policy, config);
+  }
+
+  
+  private static final Method SLEEP_METHOD;
+  private static final int SLEEP_METHOD_SHARD_ARG_IDX = -1;  // Don't pass in shard list.
+  static {
+    try {
+      SLEEP_METHOD = ISleepServer.class.getMethod("sleep", 
+              new Class[] { Long.TYPE, Integer.TYPE });
+    } catch (NoSuchMethodException e) {
+      throw new RuntimeException("Could not find method sleep() in ISleepServer!");
+    }
+  }
+  
+  public void sleep(final long msec, final String[] indexNames) throws KattaException {
+    sleep(msec, 0, indexNames);
+  }
+  
+  public void sleep(final long msec, final int delta, final String[] indexNames) throws KattaException {
+    kattaClient.broadcastToIndices(SLEEP_METHOD, SLEEP_METHOD_SHARD_ARG_IDX, indexNames, msec, delta);
+  }
+
+  
+  public void close() {
+    kattaClient.close();
+  }
+
+}
Index: src/test/java/net/sf/katta/util/ISleepServer.java
===================================================================
--- src/test/java/net/sf/katta/util/ISleepServer.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/test/java/net/sf/katta/util/ISleepServer.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,35 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.util;
+
+import org.apache.hadoop.ipc.VersionedProtocol;
+
+/**
+ * The public interface for the back end of a dummy server that just
+ * sleeps for a while then returns null. Used for testing.
+ */
+public interface ISleepServer extends VersionedProtocol {
+  
+  /**
+   * Sleep for the given number of milliseconds, +- a random delta.
+   * @param msec How long each node should sleep for.
+   * @param delta The maximum size of the delta (msec) to add or remove
+   *     from the specified time. The node will choose an evenly distributed
+   *     random sleep time from msec-delta to msec+delta.
+   */
+  public void sleep(long msec, int delta);
+
+}
Index: src/test/java/net/sf/katta/util/GenerateMapFiles.java
===================================================================
--- src/test/java/net/sf/katta/util/GenerateMapFiles.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/test/java/net/sf/katta/util/GenerateMapFiles.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,89 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.util;
+
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.MapFile;
+import org.apache.hadoop.io.Text;
+
+public class GenerateMapFiles {
+
+  /**
+   * This generates the very simple MapFiles in katta/src/test/testMapFile[AB]/.
+   * These files are supposed to simulate taking 2 large MapFiles and splitting the first one
+   * into 4 shards, the second into 2 shards. We do not provide such a tool yet.
+   * The results are checked in, so you should not need to run this. Is is provided
+   * as a reference.
+   */
+  public static void main(String[] args) throws Exception {
+    Configuration conf = new Configuration();
+    conf.set("io.file.buffer.size", "4096");
+    FileSystem fs = new RawLocalFileSystem();
+    fs.setConf(conf);
+    //
+    File f = new File("src/test/testMapFileA/a1");
+    MapFile.Writer w = new MapFile.Writer(conf, fs, f.getAbsolutePath(), Text.class, Text.class);
+    write(w, "a.txt", "This is a test");
+    write(w, "b.xml", "<name>test</name>");
+    write(w, "c.log", "1/1/2009: test");
+    w.close();
+    //
+    f = new File("src/test/testMapFileA/a2");
+    w = new MapFile.Writer(conf, fs, f.getAbsolutePath(), Text.class, Text.class);
+    write(w, "d.html", "<b>test</b>");
+    write(w, "e.txt", "An e test");
+    write(w, "f.log", "1/2/2009: test2");
+    w.close();
+    //
+    f = new File("src/test/testMapFileA/a3");
+    w = new MapFile.Writer(conf, fs, f.getAbsolutePath(), Text.class, Text.class);
+    write(w, "g.log", "1/3/2009: more test");
+    write(w, "h.txt", "Test in part 3");
+    w.close();
+    //
+    f = new File("src/test/testMapFileA/a4");
+    w = new MapFile.Writer(conf, fs, f.getAbsolutePath(), Text.class, Text.class);
+    write(w, "i.xml", "<i>test</i>");
+    write(w, "j.log", "1/4/2009: 4 test");
+    write(w, "k.out", "test data");
+    write(w, "l.txt", "line 4");
+    w.close();
+    //
+    //
+    f = new File("src/test/testMapFileB/b1");
+    w = new MapFile.Writer(conf, fs, f.getAbsolutePath(), Text.class, Text.class);
+    write(w, "u.txt", "Test U text");
+    write(w, "v.xml", "<victor>foo</victor>");
+    write(w, "w.txt", "where is test");
+    w.close();
+    //
+    f = new File("src/test/testMapFileB/b2");
+    w = new MapFile.Writer(conf, fs, f.getAbsolutePath(), Text.class, Text.class);
+    write(w, "x.txt", "xrays ionize");
+    write(w, "y.xml", "<yankee>foo</yankee>");
+    write(w, "z.xml", "<zed>foo</zed>");
+    w.close();
+  }
+  
+  private static void write(MapFile.Writer w, String fn, String data) throws Exception {
+    w.append(new Text(fn), new Text(data));
+  }
+  
+}
Index: src/test/java/net/sf/katta/client/ClientFailoverTest.java
===================================================================
--- src/test/java/net/sf/katta/client/ClientFailoverTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/test/java/net/sf/katta/client/ClientFailoverTest.java	(.../trunk/katta)	(revision 10997)
@@ -1,184 +0,0 @@
-/**
- * Copyright 2008 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.sf.katta.client;
-
-import java.util.List;
-
-import net.sf.katta.AbstractKattaTest;
-import net.sf.katta.Katta;
-import net.sf.katta.node.Hits;
-import net.sf.katta.node.Query;
-import net.sf.katta.zk.ZKClient;
-import net.sf.katta.zk.ZkPathes;
-
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-
-public class ClientFailoverTest extends AbstractKattaTest {
-
-  MasterStartThread masterThread;
-  NodeStartThread nodeThread1;
-  NodeStartThread nodeThread2;
-  String index = "index1";
-
-  static int nodePort1 = 20000;
-  static int nodePort2 = 20001;
-
-  @Override
-  protected void onSetUp2() throws Exception {
-    masterThread = startMaster();
-    nodeThread1 = startNode(nodePort1);
-    nodeThread2 = startNode("/tmp/kattaShards2", nodePort2);
-
-    masterThread.join();
-    nodeThread1.join();
-    nodeThread2.join();
-
-    // distribute index over 2 nodes
-    Katta katta = new Katta();
-    katta.addIndex(index, "src/test/testIndexA", StandardAnalyzer.class.getName(), 2);
-    katta.close();
-  }
-
-  @Override
-  protected void onTearDown() throws Exception {
-    masterThread.shutdown();
-    // jz: since hadoop18 the ipc acts a little fragile when starting and
-    // stopping ipc-servers rapidly on the same port so we increment port
-    // numbers from test to test
-    nodePort1 = nodePort1 + 4;
-    nodePort2 = nodePort2 + 4;
-  }
-
-  public void testSearch_NodeProxyDownAfterClientInitialization() throws Exception {
-    // start search client
-    Client searchClient = new Client();
-
-    // shutdown proxy of node1
-    nodeThread1.getNode().getRpcServer().stop();
-
-    final Query query = new Query("content:the");
-    System.out.println("=========================");
-    assertSearchResults(10, searchClient.search(query, new String[] { index }, 10));
-    assertSearchResults(10, searchClient.search(query, new String[] { index }, 10));
-    // search 2 time to ensure we get all availible nodes
-    System.out.println("=========================");
-
-    nodeThread1.shutdown();
-    nodeThread2.shutdown();
-    searchClient.close();
-  }
-
-  public void testCount_NodeProxyDownAfterClientInitialization() throws Exception {
-    // start search client
-    Client searchClient = new Client();
-
-    // shutdown proxy of node1
-    nodeThread1.getNode().getRpcServer().stop();
-
-    final Query query = new Query("content:the");
-    System.out.println("=========================");
-    assertEquals(937, searchClient.count(query, new String[] { index }));
-    assertEquals(937, searchClient.count(query, new String[] { index }));
-    // search 2 time to ensure we get all availible nodes
-    System.out.println("=========================");
-
-    nodeThread1.shutdown();
-    nodeThread2.shutdown();
-    searchClient.close();
-  }
-
-  public void testGetDetails_NodeProxyDownAfterClientInitialization() throws Exception {
-    // start search client
-    Client searchClient = new Client();
-    final Query query = new Query("content:the");
-    Hits hits = searchClient.search(query, new String[] { index }, 10);
-
-    // shutdown proxy of node1
-    System.out.println("=========================");
-    if (nodeThread1.getNode().getName().equals(hits.getHits().get(0).getNode())) {
-      nodeThread1.getNode().getRpcServer().stop();
-    } else {
-      nodeThread2.getNode().getRpcServer().stop();
-    }
-    assertFalse(searchClient.getDetails(hits.getHits().get(0)).isEmpty());
-    assertFalse(searchClient.getDetails(hits.getHits().get(0)).isEmpty());
-    // search 2 time to ensure we get all availible nodes
-    System.out.println("=========================");
-
-    nodeThread1.shutdown();
-    nodeThread2.shutdown();
-    searchClient.close();
-  }
-
-  public void testAllNodeProxyDownAfterClientInitialization() throws Exception {
-    // start search client
-    Client searchClient = new Client();
-    final Query query = new Query("content:the");
-    nodeThread1.getNode().getRpcServer().stop();
-    nodeThread2.getNode().getRpcServer().stop();
-
-    System.out.println("=========================");
-    try {
-      searchClient.search(query, new String[] { index }, 10);
-      fail("should throw exception");
-    } catch (ShardAccessException e) {
-      // expected
-    }
-    System.out.println("=========================");
-
-    nodeThread1.shutdown();
-    nodeThread2.shutdown();
-    searchClient.close();
-  }
-
-  private void assertSearchResults(int expectedResults, Hits hits) {
-    assertNotNull(hits);
-    assertEquals(expectedResults, hits.getHits().size());
-  }
-
-  public void testNodeNotReachable() throws Exception {
-    // shutdown 2nd node
-    nodeThread2.shutdown();
-    waitOnNodes(masterThread, 1);
-
-    // simulate 2nd node alive and serving shard
-    String node2Name = nodeThread2.getNode().getName();
-    ZKClient zkClient = masterThread.getZkClient();
-    List<String> shards = zkClient.getChildren(ZkPathes.getIndexPath(index));
-    zkClient.create(ZkPathes.getNodePath(node2Name));
-    for (String shard : shards) {
-      zkClient.create(ZkPathes.getShard2NodePath(shard, node2Name));
-    }
-    waitOnNodes(masterThread, 2);
-
-    // start search client
-    Client searchClient = new Client();
-    final Query query = new Query("content:the");
-    assertSearchResults(10, searchClient.search(query, new String[] { index }, 10));
-    assertSearchResults(10, searchClient.search(query, new String[] { index }, 10));
-
-    // flip node1/node2 alive status
-    nodeThread2 = startNode("/tmp/kattaShards2", nodePort2);
-    nodeThread2.join();
-    nodeThread1.shutdown();
-    waitOnNodes(masterThread, 1);
-
-    // search again
-    assertSearchResults(10, searchClient.search(query, new String[] { index }, 10));
-    searchClient.close();
-  }
-
-}
Index: src/test/java/net/sf/katta/client/ClientTest.java
===================================================================
--- src/test/java/net/sf/katta/client/ClientTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/test/java/net/sf/katta/client/ClientTest.java	(.../trunk/katta)	(revision 10997)
@@ -1,201 +0,0 @@
-/**
- * Copyright 2008 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.sf.katta.client;
-
-import java.util.List;
-import java.util.Set;
-
-import net.sf.katta.AbstractKattaTest;
-import net.sf.katta.master.Master;
-import net.sf.katta.node.Hit;
-import net.sf.katta.node.Hits;
-import net.sf.katta.node.Node;
-import net.sf.katta.testutil.TestResources;
-import net.sf.katta.util.KattaException;
-
-import org.apache.hadoop.io.MapWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.log4j.Logger;
-import org.apache.lucene.analysis.KeywordAnalyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.Query;
-
-/**
- * Test for {@link Client}.
- */
-public class ClientTest extends AbstractKattaTest {
-
-  private static Logger LOG = Logger.getLogger(ClientTest.class);
-
-  private static final String INDEX1 = "index1";
-  private static final String INDEX2 = "index2";
-  private static final String INDEX3 = "index3";
-
-  private static Node _node1;
-  private static Node _node2;
-  private static Master _master;
-  private static IDeployClient _deployClient;
-  private static IClient _client;
-
-  public ClientTest() {
-    super(false);
-  }
-
-  @Override
-  protected void onBeforeClass() throws Exception {
-    MasterStartThread masterStartThread = startMaster();
-    _master = masterStartThread.getMaster();
-
-    NodeStartThread nodeStartThread1 = startNode();
-    NodeStartThread nodeStartThread2 = startNode();
-    _node1 = nodeStartThread1.getNode();
-    _node2 = nodeStartThread2.getNode();
-    masterStartThread.join();
-    nodeStartThread1.join();
-    nodeStartThread2.join();
-    waitOnNodes(masterStartThread, 2);
-
-    _deployClient = new DeployClient(_conf);
-    _deployClient.addIndex(INDEX1, TestResources.INDEX1.getAbsolutePath(), StandardAnalyzer.class.getName(), 1)
-        .joinDeployment();
-    _deployClient.addIndex(INDEX2, TestResources.INDEX1.getAbsolutePath(), StandardAnalyzer.class.getName(), 1)
-        .joinDeployment();
-    _deployClient.addIndex(INDEX3, TestResources.INDEX1.getAbsolutePath(), StandardAnalyzer.class.getName(), 1)
-        .joinDeployment();
-    _client = new Client();
-  }
-
-  @Override
-  protected void onAfterClass() throws Exception {
-    _client.close();
-    _deployClient.disconnect();
-    _node1.shutdown();
-    _node2.shutdown();
-    _master.shutdown();
-  }
-
-  public void testCount() throws KattaException, ParseException {
-    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("content: the");
-    final int count = _client.count(query, new String[] { INDEX1 });
-    assertEquals(937, count);
-  }
-
-  public void testGetDetails() throws KattaException, ParseException {
-    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("content: the");
-    final Hits hits = _client.search(query, new String[] { INDEX1 }, 10);
-    assertNotNull(hits);
-    assertEquals(10, hits.getHits().size());
-    for (final Hit hit : hits.getHits()) {
-      final MapWritable details = _client.getDetails(hit);
-      final Set<Writable> keySet = details.keySet();
-      assertFalse(keySet.isEmpty());
-      final Writable writable = details.get(new Text("path"));
-      assertNotNull(writable);
-    }
-  }
-
-  public void testGetDetailsConcurrently() throws KattaException, ParseException, InterruptedException {
-    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("content: the");
-    final Hits hits = _client.search(query, new String[] { INDEX1 }, 10);
-    assertNotNull(hits);
-    assertEquals(10, hits.getHits().size());
-    List<MapWritable> detailList = _client.getDetails(hits.getHits());
-    assertEquals(hits.getHits().size(), detailList.size());
-    for (int i = 0; i < detailList.size(); i++) {
-      final MapWritable details1 = _client.getDetails(hits.getHits().get(i));
-      final MapWritable details2 = detailList.get(i);
-      assertEquals(details1.entrySet(), details2.entrySet());
-      final Set<Writable> keySet = details2.keySet();
-      assertFalse(keySet.isEmpty());
-      final Writable writable = details2.get(new Text("path"));
-      assertNotNull(writable);
-    }
-  }
-
-  public void testSearch() throws KattaException, ParseException {
-    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("foo: bar");
-    float currentQueryPerMinute = _client.getQueryPerMinute();
-    final Hits hits = _client.search(query, new String[] { INDEX3, INDEX2 });
-    assertNotNull(hits);
-    assertEquals(currentQueryPerMinute + 1, _client.getQueryPerMinute());
-    for (final Hit hit : hits.getHits()) {
-      writeToLog(hit);
-    }
-    assertEquals(8, hits.size());
-    assertEquals(8, hits.getHits().size());
-    for (final Hit hit : hits.getHits()) {
-      LOG.info(hit.getNode() + " -- " + hit.getScore() + " -- " + hit.getDocId());
-    }
-  }
-
-  public void testSearchLimit() throws KattaException, ParseException {
-    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("foo: bar");
-    final Hits hits = _client.search(query, new String[] { INDEX3, INDEX2 }, 1);
-    assertNotNull(hits);
-    for (final Hit hit : hits.getHits()) {
-      writeToLog(hit);
-    }
-    assertEquals(8, hits.size());
-    assertEquals(1, hits.getHits().size());
-    for (final Hit hit : hits.getHits()) {
-      LOG.info(hit.getNode() + " -- " + hit.getScore() + " -- " + hit.getDocId());
-    }
-  }
-
-  public void testKatta20SearchLimitMaxNumberOfHits() throws KattaException, ParseException {
-    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("foo: bar");
-    final Hits expectedHits = _client.search(query, new String[] { INDEX1 }, 4);
-    assertNotNull(expectedHits);
-    LOG.info("Expected hits:");
-    for (final Hit hit : expectedHits.getHits()) {
-      writeToLog(hit);
-    }
-    assertEquals(4, expectedHits.getHits().size());
-
-    for (int i = 0; i < 1000; i++) {
-      // Now we redo the search, but limit the max number of hits. We expect the same
-      // ordering of hits.
-      for (int maxHits = 1; maxHits < expectedHits.size() + 1; maxHits++) {
-        final Hits hits = _client.search(query, new String[] { INDEX1 }, maxHits);
-        assertNotNull(hits);
-        assertEquals(maxHits, hits.getHits().size());
-        for (int j = 0; j < hits.getHits().size(); j++) {
-//           writeToLog("expected: ", expectedHits.getHits().get(j));
-//           writeToLog("actual : ", hits.getHits().get(j));
-          assertEquals(expectedHits.getHits().get(j).getScore(), hits.getHits().get(j).getScore());
-        }
-      }
-    }
-  }
-
-  private void writeToLog(Hit hit) {
-    LOG.info(hit.getNode() + " -- " + hit.getShard() + " -- " + hit.getScore() + " -- " + hit.getDocId());
-  }
-
-  public void testSearchSimiliarity() throws KattaException, ParseException {
-    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("foo: bar");
-    final Hits hits = _client.search(query, new String[] { INDEX2 });
-    assertNotNull(hits);
-    assertEquals(4, hits.getHits().size());
-    for (final Hit hit : hits.getHits()) {
-      LOG.info(hit.getNode() + " -- " + hit.getScore() + " -- " + hit.getDocId());
-    }
-  }
-
-}
Index: src/test/java/net/sf/katta/client/LuceneClientFailoverTest.java
===================================================================
--- src/test/java/net/sf/katta/client/LuceneClientFailoverTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/test/java/net/sf/katta/client/LuceneClientFailoverTest.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,183 @@
+/**
+ * Copyright 2008 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.client;
+
+import java.util.List;
+
+import net.sf.katta.AbstractKattaTest;
+import net.sf.katta.Katta;
+import net.sf.katta.node.Hits;
+import net.sf.katta.node.LuceneServer;
+import net.sf.katta.node.Query;
+import net.sf.katta.zk.ZKClient;
+import net.sf.katta.zk.ZkPathes;
+
+public class LuceneClientFailoverTest extends AbstractKattaTest {
+
+  MasterStartThread masterThread;
+  NodeStartThread nodeThread1;
+  NodeStartThread nodeThread2;
+  String index = "index1";
+
+  static int nodePort1 = 20000;
+  static int nodePort2 = 20001;
+
+  @Override
+  protected void onSetUp2() throws Exception {
+    masterThread = startMaster();
+    nodeThread1 = startNode(new LuceneServer(), nodePort1);
+    nodeThread2 = startNode(new LuceneServer(), nodePort2, "/tmp/kattaShards2");
+
+    masterThread.join();
+    nodeThread1.join();
+    nodeThread2.join();
+
+    // distribute index over 2 nodes
+    Katta katta = new Katta();
+    katta.addIndex(index, "src/test/testIndexA", 2);
+    katta.close();
+  }
+
+  @Override
+  protected void onTearDown() throws Exception {
+    masterThread.shutdown();
+    // jz: since hadoop18 the ipc acts a little fragile when starting and
+    // stopping ipc-servers rapidly on the same port so we increment port
+    // numbers from test to test
+    nodePort1 = nodePort1 + 4;
+    nodePort2 = nodePort2 + 4;
+  }
+
+  public void testSearch_NodeProxyDownAfterClientInitialization() throws Exception {
+    // start search client
+    LuceneClient searchClient = new LuceneClient();
+
+    // shutdown proxy of node1
+    nodeThread1.getNode().getRpcServer().stop();
+
+    final Query query = new Query("content:the");
+    System.out.println("=========================");
+    assertSearchResults(10, searchClient.search(query, new String[] { index }, 10));
+    assertSearchResults(10, searchClient.search(query, new String[] { index }, 10));
+    // search 2 time to ensure we get all availible nodes
+    System.out.println("=========================");
+
+    nodeThread1.shutdown();
+    nodeThread2.shutdown();
+    searchClient.close();
+  }
+
+  public void testCount_NodeProxyDownAfterClientInitialization() throws Exception {
+    // start search client
+    LuceneClient searchClient = new LuceneClient();
+
+    // shutdown proxy of node1
+    nodeThread1.getNode().getRpcServer().stop();
+
+    final Query query = new Query("content:the");
+    System.out.println("=========================");
+    assertEquals(937, searchClient.count(query, new String[] { index }));
+    assertEquals(937, searchClient.count(query, new String[] { index }));
+    // search 2 time to ensure we get all availible nodes
+    System.out.println("=========================");
+
+    nodeThread1.shutdown();
+    nodeThread2.shutdown();
+    searchClient.close();
+  }
+
+  public void testGetDetails_NodeProxyDownAfterClientInitialization() throws Exception {
+    // start search client
+    LuceneClient searchClient = new LuceneClient();
+    final Query query = new Query("content:the");
+    Hits hits = searchClient.search(query, new String[] { index }, 10);
+
+    // shutdown proxy of node1
+    System.out.println("=========================");
+    if (nodeThread1.getNode().getName().equals(hits.getHits().get(0).getNode())) {
+      nodeThread1.getNode().getRpcServer().stop();
+    } else {
+      nodeThread2.getNode().getRpcServer().stop();
+    }
+    assertFalse(searchClient.getDetails(hits.getHits().get(0)).isEmpty());
+    assertFalse(searchClient.getDetails(hits.getHits().get(0)).isEmpty());
+    // search 2 time to ensure we get all availible nodes
+    System.out.println("=========================");
+
+    nodeThread1.shutdown();
+    nodeThread2.shutdown();
+    searchClient.close();
+  }
+
+  public void testAllNodeProxyDownAfterClientInitialization() throws Exception {
+    // start search client
+    LuceneClient searchClient = new LuceneClient();
+    final Query query = new Query("content:the");
+    nodeThread1.getNode().getRpcServer().stop();
+    nodeThread2.getNode().getRpcServer().stop();
+
+    System.out.println("=========================");
+    try {
+      searchClient.search(query, new String[] { index }, 10);
+      fail("should throw exception");
+    } catch (ShardAccessException e) {
+      // expected
+    }
+    System.out.println("=========================");
+
+    nodeThread1.shutdown();
+    nodeThread2.shutdown();
+    searchClient.close();
+  }
+
+  private void assertSearchResults(int expectedResults, Hits hits) {
+    assertNotNull(hits);
+    assertEquals(expectedResults, hits.getHits().size());
+  }
+
+  public void testNodeNotReachable() throws Exception {
+    // shutdown 2nd node
+    nodeThread2.shutdown();
+    waitOnNodes(masterThread, 1);
+
+    // simulate 2nd node alive and serving shard
+    String node2Name = nodeThread2.getNode().getName();
+    ZKClient zkClient = masterThread.getZkClient();
+    List<String> shards = zkClient.getChildren(ZkPathes.getIndexPath(index));
+    zkClient.create(ZkPathes.getNodePath(node2Name));
+    for (String shard : shards) {
+      zkClient.create(ZkPathes.getShard2NodePath(shard, node2Name));
+    }
+    waitOnNodes(masterThread, 2);
+
+    // start search client
+    LuceneClient searchClient = new LuceneClient();
+    final Query query = new Query("content:the");
+    assertSearchResults(10, searchClient.search(query, new String[] { index }, 10));
+    assertSearchResults(10, searchClient.search(query, new String[] { index }, 10));
+
+    // flip node1/node2 alive status
+    nodeThread2 = startNode(new LuceneServer(), nodePort2, "/tmp/kattaShards2");
+    nodeThread2.join();
+    nodeThread1.shutdown();
+    waitOnNodes(masterThread, 1);
+
+    // search again
+    assertSearchResults(10, searchClient.search(query, new String[] { index }, 10));
+    searchClient.close();
+  }
+
+}
Index: src/test/java/net/sf/katta/client/SleepClientTest.java
===================================================================
--- src/test/java/net/sf/katta/client/SleepClientTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/test/java/net/sf/katta/client/SleepClientTest.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,119 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.client;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import net.sf.katta.AbstractKattaTest;
+import net.sf.katta.master.Master;
+import net.sf.katta.node.Node;
+import net.sf.katta.testutil.TestResources;
+import net.sf.katta.util.ISleepClient;
+import net.sf.katta.util.KattaException;
+import net.sf.katta.util.SleepClient;
+import net.sf.katta.util.SleepServer;
+
+import org.apache.log4j.Logger;
+
+/**
+ * Test for {@link SleepClient}.
+ */
+public class SleepClientTest extends AbstractKattaTest {
+
+  @SuppressWarnings("unused")
+  private static Logger LOG = Logger.getLogger(SleepClientTest.class);
+
+  private static final String INDEX1 = "index1";
+  
+  private static final String[] INDEX_1 = { INDEX1 };
+
+  private static Node _node1;
+  private static Master _master;
+  private static IDeployClient _deployClient;
+  private static ISleepClient _client;
+
+  public SleepClientTest() {
+    super(false);
+  }
+
+  @Override
+  protected void onBeforeClass() throws Exception {
+    MasterStartThread masterStartThread = startMaster();
+    _master = masterStartThread.getMaster();
+
+    NodeStartThread nodeStartThread1 = startNode(new SleepServer());
+    _node1 = nodeStartThread1.getNode();
+    masterStartThread.join();
+    nodeStartThread1.join();
+    waitOnNodes(masterStartThread, 1);
+
+    _deployClient = new DeployClient(_conf);
+    _deployClient.addIndex(INDEX1, TestResources.MAP_FILE_A.getAbsolutePath(), 1).joinDeployment();
+    _client = new SleepClient();
+  }
+
+  @Override
+  protected void onAfterClass() throws Exception {
+    _client.close();
+    _deployClient.disconnect();
+    _node1.shutdown();
+    _master.shutdown();
+  }
+
+  public void testDelay() throws KattaException {
+    long start = System.currentTimeMillis();
+    _client.sleep(0, INDEX_1);
+    long d1 = System.currentTimeMillis() - start;
+    start = System.currentTimeMillis();
+    _client.sleep(1000, INDEX_1);
+    long d2 = System.currentTimeMillis() - start;
+    assertTrue(d2 - d1 > 200);
+  }
+  
+  public void testMultiThreadedAccess() throws Exception {
+    Random rand = new Random("sleepy".hashCode());
+    List<Thread> threads = new ArrayList<Thread>();
+    final List<Exception> exceptions = new ArrayList<Exception>();
+    long startTime = System.currentTimeMillis();
+    for (int i=0; i<10; i++) {
+      final Random rand2 = new Random(rand.nextInt());
+      Thread t = new Thread(new Runnable() {
+        public void run() {
+          for (int j=0; j<50; j++) {
+            int n = rand2.nextInt(20);
+            try {
+              _client.sleep(n, INDEX_1);
+            } catch (Exception e) {
+              System.err.println(e);
+              exceptions.add(e);
+              break;
+            }
+          }
+        }
+      });
+      threads.add(t);
+      t.start();
+    }
+    for (Thread t : threads) {
+      t.join();
+    }
+    System.out.println("Took " + (System.currentTimeMillis() - startTime) + " msec.");
+    assertTrue(exceptions.isEmpty());
+  }
+
+}
Index: src/test/java/net/sf/katta/client/LuceneClientTest.java
===================================================================
--- src/test/java/net/sf/katta/client/LuceneClientTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/test/java/net/sf/katta/client/LuceneClientTest.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,199 @@
+/**
+ * Copyright 2008 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.client;
+
+import java.util.List;
+import java.util.Set;
+
+import net.sf.katta.AbstractKattaTest;
+import net.sf.katta.master.Master;
+import net.sf.katta.node.Hit;
+import net.sf.katta.node.Hits;
+import net.sf.katta.node.LuceneServer;
+import net.sf.katta.node.Node;
+import net.sf.katta.testutil.TestResources;
+import net.sf.katta.util.KattaException;
+
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.log4j.Logger;
+import org.apache.lucene.analysis.KeywordAnalyzer;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.Query;
+
+/**
+ * Test for {@link LuceneClient}.
+ */
+public class LuceneClientTest extends AbstractKattaTest {
+
+  private static Logger LOG = Logger.getLogger(LuceneClientTest.class);
+
+  private static final String INDEX1 = "index1";
+  private static final String INDEX2 = "index2";
+  private static final String INDEX3 = "index3";
+
+  private static Node _node1;
+  private static Node _node2;
+  private static Master _master;
+  private static IDeployClient _deployClient;
+  private static ILuceneClient _client;
+
+  public LuceneClientTest() {
+    super(false);
+  }
+
+  @Override
+  protected void onBeforeClass() throws Exception {
+    MasterStartThread masterStartThread = startMaster();
+    _master = masterStartThread.getMaster();
+
+    NodeStartThread nodeStartThread1 = startNode(new LuceneServer());
+    NodeStartThread nodeStartThread2 = startNode(new LuceneServer());
+    _node1 = nodeStartThread1.getNode();
+    _node2 = nodeStartThread2.getNode();
+    masterStartThread.join();
+    nodeStartThread1.join();
+    nodeStartThread2.join();
+    waitOnNodes(masterStartThread, 2);
+
+    _deployClient = new DeployClient(_conf);
+    _deployClient.addIndex(INDEX1, TestResources.INDEX1.getAbsolutePath(), 1)
+        .joinDeployment();
+    _deployClient.addIndex(INDEX2, TestResources.INDEX1.getAbsolutePath(), 1)
+        .joinDeployment();
+    _deployClient.addIndex(INDEX3, TestResources.INDEX1.getAbsolutePath(), 1)
+        .joinDeployment();
+    _client = new LuceneClient();
+  }
+
+  @Override
+  protected void onAfterClass() throws Exception {
+    _client.close();
+    _deployClient.disconnect();
+    _node1.shutdown();
+    _node2.shutdown();
+    _master.shutdown();
+  }
+
+  public void testCount() throws KattaException, ParseException {
+    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("content: the");
+    final int count = _client.count(query, new String[] { INDEX1 });
+    assertEquals(937, count);
+  }
+
+  public void testGetDetails() throws KattaException, ParseException {
+    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("content: the");
+    final Hits hits = _client.search(query, new String[] { INDEX1 }, 10);
+    assertNotNull(hits);
+    assertEquals(10, hits.getHits().size());
+    for (final Hit hit : hits.getHits()) {
+      final MapWritable details = _client.getDetails(hit);
+      final Set<Writable> keySet = details.keySet();
+      assertFalse(keySet.isEmpty());
+      final Writable writable = details.get(new Text("path"));
+      assertNotNull(writable);
+    }
+  }
+
+  public void testGetDetailsConcurrently() throws KattaException, ParseException, InterruptedException {
+    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("content: the");
+    final Hits hits = _client.search(query, new String[] { INDEX1 }, 10);
+    assertNotNull(hits);
+    assertEquals(10, hits.getHits().size());
+    List<MapWritable> detailList = _client.getDetails(hits.getHits());
+    assertEquals(hits.getHits().size(), detailList.size());
+    for (int i = 0; i < detailList.size(); i++) {
+      final MapWritable details1 = _client.getDetails(hits.getHits().get(i));
+      final MapWritable details2 = detailList.get(i);
+      assertEquals(details1.entrySet(), details2.entrySet());
+      final Set<Writable> keySet = details2.keySet();
+      assertFalse(keySet.isEmpty());
+      final Writable writable = details2.get(new Text("path"));
+      assertNotNull(writable);
+    }
+  }
+
+  public void testSearch() throws KattaException, ParseException {
+    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("foo: bar");
+    final Hits hits = _client.search(query, new String[] { INDEX3, INDEX2 });
+    assertNotNull(hits);
+    for (final Hit hit : hits.getHits()) {
+      writeToLog(hit);
+    }
+    assertEquals(8, hits.size());
+    assertEquals(8, hits.getHits().size());
+    for (final Hit hit : hits.getHits()) {
+      LOG.info(hit.getNode() + " -- " + hit.getScore() + " -- " + hit.getDocId());
+    }
+  }
+
+  public void testSearchLimit() throws KattaException, ParseException {
+    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("foo: bar");
+    final Hits hits = _client.search(query, new String[] { INDEX3, INDEX2 }, 1);
+    assertNotNull(hits);
+    for (final Hit hit : hits.getHits()) {
+      writeToLog(hit);
+    }
+    assertEquals(8, hits.size());
+    assertEquals(1, hits.getHits().size());
+    for (final Hit hit : hits.getHits()) {
+      LOG.info(hit.getNode() + " -- " + hit.getScore() + " -- " + hit.getDocId());
+    }
+  }
+
+  public void testKatta20SearchLimitMaxNumberOfHits() throws KattaException, ParseException {
+    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("foo: bar");
+    final Hits expectedHits = _client.search(query, new String[] { INDEX1 }, 4);
+    assertNotNull(expectedHits);
+    LOG.info("Expected hits:");
+    for (final Hit hit : expectedHits.getHits()) {
+      writeToLog(hit);
+    }
+    assertEquals(4, expectedHits.getHits().size());
+
+    for (int i = 0; i < 1000; i++) {
+      // Now we redo the search, but limit the max number of hits. We expect the same
+      // ordering of hits.
+      for (int maxHits = 1; maxHits < expectedHits.size() + 1; maxHits++) {
+        final Hits hits = _client.search(query, new String[] { INDEX1 }, maxHits);
+        assertNotNull(hits);
+        assertEquals(maxHits, hits.getHits().size());
+        for (int j = 0; j < hits.getHits().size(); j++) {
+//           writeToLog("expected: ", expectedHits.getHits().get(j));
+//           writeToLog("actual : ", hits.getHits().get(j));
+          assertEquals(expectedHits.getHits().get(j).getScore(), hits.getHits().get(j).getScore());
+        }
+      }
+    }
+  }
+
+  private void writeToLog(Hit hit) {
+    LOG.info(hit.getNode() + " -- " + hit.getShard() + " -- " + hit.getScore() + " -- " + hit.getDocId());
+  }
+
+  public void testSearchSimiliarity() throws KattaException, ParseException {
+    final Query query = new QueryParser("", new KeywordAnalyzer()).parse("foo: bar");
+    final Hits hits = _client.search(query, new String[] { INDEX2 });
+    assertNotNull(hits);
+    assertEquals(4, hits.getHits().size());
+    for (final Hit hit : hits.getHits()) {
+      LOG.info(hit.getNode() + " -- " + hit.getScore() + " -- " + hit.getDocId());
+    }
+  }
+
+}
Index: src/test/java/net/sf/katta/client/MapFileClientTest.java
===================================================================
--- src/test/java/net/sf/katta/client/MapFileClientTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/test/java/net/sf/katta/client/MapFileClientTest.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,178 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.client;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import net.sf.katta.AbstractKattaTest;
+import net.sf.katta.master.Master;
+import net.sf.katta.node.MapFileServer;
+import net.sf.katta.node.Node;
+import net.sf.katta.testutil.TestResources;
+import net.sf.katta.util.KattaException;
+
+import org.apache.log4j.Logger;
+
+/**
+ * Test for {@link MapFileClient}.
+ */
+public class MapFileClientTest extends AbstractKattaTest {
+
+  @SuppressWarnings("unused")
+  private static Logger LOG = Logger.getLogger(MapFileClientTest.class);
+
+  private static final String INDEX1 = "index1";
+  private static final String INDEX2 = "index2";
+  
+  private static final String[] INDEX_1 = { INDEX1 };
+  private static final String[] INDEX_2 = { INDEX2 };
+  private static final String[] INDEX_BOTH = { INDEX1, INDEX2 };
+
+  private static Node _node1;
+  private static Node _node2;
+  private static Master _master;
+  private static IDeployClient _deployClient;
+  private static IMapFileClient _client;
+
+  public MapFileClientTest() {
+    super(false);
+  }
+
+  @Override
+  protected void onBeforeClass() throws Exception {
+    MasterStartThread masterStartThread = startMaster();
+    _master = masterStartThread.getMaster();
+
+    NodeStartThread nodeStartThread1 = startNode(new MapFileServer());
+    NodeStartThread nodeStartThread2 = startNode(new MapFileServer());
+    _node1 = nodeStartThread1.getNode();
+    _node2 = nodeStartThread2.getNode();
+    masterStartThread.join();
+    nodeStartThread1.join();
+    nodeStartThread2.join();
+    waitOnNodes(masterStartThread, 2);
+
+    _deployClient = new DeployClient(_conf);
+    _deployClient.addIndex(INDEX1, TestResources.MAP_FILE_A.getAbsolutePath(), 1).joinDeployment();
+    _deployClient.addIndex(INDEX2, TestResources.MAP_FILE_B.getAbsolutePath(), 1).joinDeployment();
+    _client = new MapFileClient();
+  }
+
+  @Override
+  protected void onAfterClass() throws Exception {
+    _client.close();
+    _deployClient.disconnect();
+    _node1.shutdown();
+    _node2.shutdown();
+    _master.shutdown();
+  }
+
+  public void testGetA() throws KattaException {
+    assertEquals("This is a test", getOneResult("a.txt", INDEX_1));
+    assertEquals("1/2/2009: test2", getOneResult("f.log", INDEX_1));
+    assertEquals("1/3/2009: more test", getOneResult("g.log", INDEX_1));
+    assertEquals("<i>test</i>", getOneResult("i.xml", INDEX_1));
+    assertMissing("u.txt", INDEX_1);
+    assertMissing("x.txt", INDEX_1);
+    assertMissing("not-found", INDEX_1);
+  }
+
+  public void testGetB() throws KattaException {
+    assertEquals("Test U text", getOneResult("u.txt", INDEX_2));
+    assertEquals("xrays ionize", getOneResult("x.txt", INDEX_2));
+    assertMissing("a.txt", INDEX_2);
+    assertMissing("f.log", INDEX_2);
+    assertMissing("g.log", INDEX_2);
+    assertMissing("i.xml", INDEX_2);
+    assertMissing("not-found", INDEX_2);
+  }
+
+  public void testGetBoth() throws KattaException {
+    assertEquals("This is a test", getOneResult("a.txt", INDEX_BOTH));
+    assertEquals("1/2/2009: test2", getOneResult("f.log", INDEX_BOTH));
+    assertEquals("1/3/2009: more test", getOneResult("g.log", INDEX_BOTH));
+    assertEquals("<i>test</i>", getOneResult("i.xml", INDEX_BOTH));
+    assertEquals("Test U text", getOneResult("u.txt", INDEX_BOTH));
+    assertEquals("xrays ionize", getOneResult("x.txt", INDEX_BOTH));
+    assertMissing("not-found", INDEX_BOTH);
+  }
+
+  public void testMultiThreadedAccess() throws Exception {
+    final Map<String, String> entries = new HashMap<String, String>();
+    entries.put("a.txt", "This is a test");
+    entries.put("b.xml", "<name>test</name>");
+    entries.put("d.html", "<b>test</b>");
+    entries.put("h.txt", "Test in part 3");
+    entries.put("i.xml", "<i>test</i>");
+    entries.put("k.out", "test data");
+    entries.put("w.txt", "where is test");
+    entries.put("x.txt", "xrays ionize");
+    entries.put("z.xml", "<zed>foo</zed>");
+    final List<String> keys = new ArrayList<String>(entries.keySet());
+    Random rand = new Random("Katta".hashCode());
+    List<Thread> threads = new ArrayList<Thread>();
+    final List<Exception> exceptions = new ArrayList<Exception>();
+    long startTime = System.currentTimeMillis();
+    final AtomicInteger count = new AtomicInteger(0);
+    for (int i=0; i<15; i++) {
+      final Random rand2 = new Random(rand.nextInt());
+      Thread t = new Thread(new Runnable() {
+        public void run() {
+          for (int j=0; j<300; j++) {
+            int n = rand2.nextInt(entries.size());
+            String key = keys.get(n);
+            try {
+              assertEquals(entries.get(key), getOneResult(key, INDEX_BOTH));
+              count.incrementAndGet();
+            } catch (Exception e) {
+              System.err.println(e);
+              exceptions.add(e);
+              break;
+            }
+          }
+        }
+      });
+      threads.add(t);
+      t.start();
+    }
+    for (Thread t : threads) {
+      t.join();
+    }
+    long time = System.currentTimeMillis() - startTime;
+    System.out.println((1000.0 * (double) count.intValue() / (double) time) + " requests / sec");
+    assertTrue(exceptions.isEmpty());
+  }
+
+  
+  private String getOneResult(String key, String[] indices) throws KattaException {
+    List<String> data = _client.get(key, indices);
+    assertNotNull(data);
+    assertEquals(1, data.size());
+    return data.get(0);
+  }
+  
+  private void assertMissing(String key, String[] indices) throws KattaException {
+    List<String> data = _client.get(key, indices);
+    assertNotNull(data);
+    assertTrue(data.isEmpty());
+  }
+
+}
Index: src/test/integration/net/sf/katta/integrationTest/KattaMiniCluster.java
===================================================================
--- src/test/integration/net/sf/katta/integrationTest/KattaMiniCluster.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/test/integration/net/sf/katta/integrationTest/KattaMiniCluster.java	(.../trunk/katta)	(revision 10997)
@@ -20,6 +20,7 @@
 import net.sf.katta.client.DeployClient;
 import net.sf.katta.client.IDeployClient;
 import net.sf.katta.master.Master;
+import net.sf.katta.node.LuceneServer;
 import net.sf.katta.node.Node;
 import net.sf.katta.util.KattaException;
 import net.sf.katta.util.NodeConfiguration;
@@ -46,7 +47,7 @@
     for (int i = 0; i < _nodes.length; i++) {
       NodeConfiguration nodeConf = new NodeConfiguration();
       nodeConf.setShardFolder(new File(nodeConf.getShardFolder(), "" + i).getAbsolutePath());
-      _nodes[i] = new Node(new ZKClient(_zkConfiguration), nodeConf);
+      _nodes[i] = new Node(new ZKClient(_zkConfiguration), nodeConf, new LuceneServer());
     }
     _master = new Master(new ZKClient(zkConfiguration));
   }
@@ -71,12 +72,11 @@
     return _nodes[i];
   }
 
-  public void deployTestIndexes(File indexFile, Class<?> analyzerClass, int deployCount, int replicationCount)
+  public void deployTestIndexes(File indexFile, int deployCount, int replicationCount)
       throws KattaException, InterruptedException {
     IDeployClient deployClient = new DeployClient(_zkConfiguration);
     for (int i = 0; i < deployCount; i++) {
-      deployClient.addIndex(indexFile.getName() + i, indexFile.getAbsolutePath(), analyzerClass.getName(),
-          replicationCount).joinDeployment();
+      deployClient.addIndex(indexFile.getName() + i, indexFile.getAbsolutePath(), replicationCount).joinDeployment();
     }
     deployClient.disconnect();
   }
Index: src/test/integration/net/sf/katta/integrationTest/SearchIntegrationTest.java
===================================================================
--- src/test/integration/net/sf/katta/integrationTest/SearchIntegrationTest.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/test/integration/net/sf/katta/integrationTest/SearchIntegrationTest.java	(.../trunk/katta)	(revision 10997)
@@ -19,8 +19,8 @@
 import java.util.List;
 
 import junit.framework.TestCase;
-import net.sf.katta.client.Client;
-import net.sf.katta.client.IClient;
+import net.sf.katta.client.ILuceneClient;
+import net.sf.katta.client.LuceneClient;
 import net.sf.katta.node.Hits;
 import net.sf.katta.node.Query;
 import net.sf.katta.testutil.TestResources;
@@ -31,7 +31,6 @@
 import net.sf.katta.util.ZkConfiguration;
 
 import org.apache.log4j.Logger;
-import org.apache.lucene.analysis.KeywordAnalyzer;
 
 public class SearchIntegrationTest extends TestCase {
 
@@ -119,7 +118,7 @@
     // start search threads
     int expectedHitCount = 12;
     SearchThread[] searchThreads = new SearchThread[25];
-    IClient searchClient = new Client();
+    ILuceneClient searchClient = new LuceneClient();
     for (int i = 0; i < searchThreads.length; i++) {
       searchThreads[i] = new SearchThread(searchClient, expectedHitCount);
       searchThreads[i].start();
@@ -181,7 +180,7 @@
     miniCluster.start();
 
     // deploy indexes
-    miniCluster.deployTestIndexes(TestResources.INDEX1, KeywordAnalyzer.class, indexCount, replicationCount);
+    miniCluster.deployTestIndexes(TestResources.INDEX1, indexCount, replicationCount);
     return miniCluster;
   }
 
@@ -196,13 +195,13 @@
     private long _firedQueryCount;
     private long _unexpectedResultCount;
 
-    private IClient _client;
+    private ILuceneClient _client;
 
     public SearchThread(long expectedTotalHitCount) {
       _expectedTotalHitCount = expectedTotalHitCount;
     }
 
-    public SearchThread(IClient client, long expectedTotalHitCount) {
+    public SearchThread(ILuceneClient client, long expectedTotalHitCount) {
       _client = client;
       _expectedTotalHitCount = expectedTotalHitCount;
     }
@@ -210,9 +209,9 @@
     @Override
     public void run() {
       try {
-        IClient client;
+        ILuceneClient client;
         if (_client == null) {
-          client = new Client();
+          client = new LuceneClient();
         } else {
           client = _client;
         }
Index: src/test/testMapFileA/a1/index
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: src/test/testMapFileA/a1/index
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Index: src/test/testMapFileA/a1/data
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: src/test/testMapFileA/a1/data
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Index: src/test/testMapFileA/a2/index
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: src/test/testMapFileA/a2/index
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Index: src/test/testMapFileA/a2/data
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: src/test/testMapFileA/a2/data
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Index: src/test/testMapFileA/a3/index
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: src/test/testMapFileA/a3/index
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Index: src/test/testMapFileA/a3/data
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: src/test/testMapFileA/a3/data
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Index: src/test/testMapFileA/a4/index
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: src/test/testMapFileA/a4/index
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Index: src/test/testMapFileA/a4/data
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream

Property changes on: src/test/testMapFileA/a4/data
___________________________________________________________________
Name: svn:mime-type
   + application/octet-stream

Index: src/main/java/net/sf/katta/Katta.java
===================================================================
--- src/main/java/net/sf/katta/Katta.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/main/java/net/sf/katta/Katta.java	(.../trunk/katta)	(revision 10997)
@@ -25,11 +25,11 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import net.sf.katta.client.Client;
 import net.sf.katta.client.DeployClient;
-import net.sf.katta.client.IClient;
 import net.sf.katta.client.IDeployClient;
 import net.sf.katta.client.IIndexDeployFuture;
+import net.sf.katta.client.ILuceneClient;
+import net.sf.katta.client.LuceneClient;
 import net.sf.katta.index.DeployedShard;
 import net.sf.katta.index.IndexMetaData;
 import net.sf.katta.index.ShardError;
@@ -39,6 +39,7 @@
 import net.sf.katta.master.Master;
 import net.sf.katta.node.Hit;
 import net.sf.katta.node.Hits;
+import net.sf.katta.node.INodeManaged;
 import net.sf.katta.node.IQuery;
 import net.sf.katta.node.Node;
 import net.sf.katta.node.NodeMetaData;
@@ -77,7 +78,7 @@
     final String command = args[0];
     // static methods first
     if (command.endsWith("startNode")) {
-      startNode();
+      startNode(args.length > 0 ? args[1] : null);
     } else if (command.endsWith("startMaster")) {
       startMaster();
     } else if (command.endsWith("version")) {
@@ -103,14 +104,14 @@
         }
       } else if (command.endsWith("addIndex")) {
         int replication = 3;
-        if (args.length < 4) {
+        if (args.length < 3) {
           printUsageAndExit();
         }
-        if (args.length == 5) {
-          replication = Integer.parseInt(args[4]);
+        if (args.length == 4) {
+          replication = Integer.parseInt(args[3]);
         }
         katta = new Katta();
-        katta.addIndex(args[1], args[2], args[3], replication);
+        katta.addIndex(args[1], args[2], replication);
       } else if (command.endsWith("removeIndex")) {
         katta = new Katta();
         katta.removeIndex(args[1]);
@@ -180,7 +181,7 @@
     try {
       removeIndex(indexName);
       Thread.sleep(5000);
-      addIndex(indexName, indexMetaData.getPath(), indexMetaData.getAnalyzerClassName(), indexMetaData
+      addIndex(indexName, indexMetaData.getPath(), indexMetaData
           .getReplicationLevel());
     } catch (InterruptedException e) {
       printError("Redeployment of index '" + indexName + "' interrupted.");
@@ -237,10 +238,33 @@
     zkServer.join();
   }
 
-  public static void startNode() throws KattaException, InterruptedException {
+  public static void startNode(String serverClassName) throws KattaException, InterruptedException {
+    INodeManaged server = null;
+    try {
+      if (serverClassName == null) {
+        serverClassName = "net.sf.katta.node.LuceneServer";
+      }
+      Class<?> serverClass = Katta.class.getClassLoader().loadClass(serverClassName);
+      if (!INodeManaged.class.isAssignableFrom(serverClass)) {
+        System.err.println("Class " + serverClassName + " does not implement INodeManaged!");
+        System.exit(1);
+      }
+      server = (INodeManaged) serverClass.newInstance();
+    } catch (ClassNotFoundException e) {
+      System.err.println("Can not find class " + serverClassName + "!");
+      System.exit(1);
+    } catch (InstantiationException e) {
+      System.err.println("Could not create instance of class " + serverClassName + "!");
+      System.exit(1);
+    } catch (IllegalAccessException e) {
+      System.err.println("Unable to access class " + serverClassName + "!");
+      System.exit(1);
+    } catch (Throwable t) {
+      throw new RuntimeException("Error getting server instance for " + serverClassName, t);
+    }
     final ZkConfiguration configuration = new ZkConfiguration();
     final ZKClient client = new ZKClient(configuration);
-    final Node node = new Node(client);
+    final Node node = new Node(client, server);
     node.start();
     Runtime.getRuntime().addShutdownHook(new Thread() {
       @Override
@@ -347,9 +371,9 @@
   public void listIndex(boolean detailedView) throws KattaException, IOException {
     final Table table;
     if (!detailedView) {
-      table = new Table(new String[] { "Name", "Status", "Path", "Shards", "Documents", "Size" });
+      table = new Table(new String[] { "Name", "Status", "Path", "Shards", "Size", "Disk Usage" });
     } else {
-      table = new Table(new String[] { "Name", "Status", "Path", "Shards", "Documents", "Size", "Analyzer",
+      table = new Table(new String[] { "Name", "Status", "Path", "Shards", "Size", "Disk Usage",
           "Replication" });
     }
 
@@ -361,13 +385,13 @@
 
       String state = metaData.getState().toString();
       List<String> shards = _zkClient.getChildren(indexZkPath);
-      int docCount = calculateDocCount(shards);
-      long indexSize = calculateIndexSize(metaData.getPath());
+      int size = calculateIndexSize(shards);
+      long indexBytes = calculateIndexDiskUsage(metaData.getPath());
       if (!detailedView) {
-        table.addRow(index, state, metaData.getPath(), shards.size(), docCount, indexSize);
+        table.addRow(index, state, metaData.getPath(), shards.size(), size, indexBytes);
       } else {
-        table.addRow(index, state, metaData.getPath(), shards.size(), docCount, indexSize, metaData
-            .getAnalyzerClassName(), metaData.getReplicationLevel());
+        table.addRow(index, state, metaData.getPath(), shards.size(), size, indexBytes,
+                metaData.getReplicationLevel());
       }
     }
     if (table.rowSize() > 0) {
@@ -377,7 +401,7 @@
     System.out.println();
   }
 
-  private long calculateIndexSize(String index) throws IOException {
+  private long calculateIndexDiskUsage(String index) throws IOException {
     Path indexPath = new Path(index);
     URI indexUri = indexPath.toUri();
     FileSystem fileSystem = FileSystem.get(indexUri, new Configuration());
@@ -387,7 +411,7 @@
     return fileSystem.getContentSummary(indexPath).getLength();
   }
 
-  private int calculateDocCount(List<String> shards) throws KattaException {
+  private int calculateIndexSize(List<String> shards) throws KattaException {
     int docCount = 0;
     for (String shard : shards) {
       List<String> deployedShards = _zkClient.getChildren(ZkPathes.getShard2NodeRootPath(shard));
@@ -400,7 +424,7 @@
     return docCount;
   }
 
-  public void addIndex(final String name, final String path, final String analyzerClass, final int replicationLevel)
+  public void addIndex(final String name, final String path, final int replicationLevel)
       throws KattaException {
     final String indexZkPath = ZkPathes.getIndexPath(name);
     if (name.trim().equals("*")) {
@@ -414,7 +438,7 @@
 
     try {
       IDeployClient deployClient = new DeployClient(_zkClient);
-      IIndexDeployFuture deployFuture = deployClient.addIndex(name, path, analyzerClass, replicationLevel);
+      IIndexDeployFuture deployFuture = deployClient.addIndex(name, path, replicationLevel);
       while (true) {
         if (deployFuture.getState() == IndexState.DEPLOYED) {
           System.out.println("deployed index " + name);
@@ -460,7 +484,7 @@
   }
 
   public static void search(final String[] indexNames, final String queryString, final int count) throws KattaException {
-    final IClient client = new Client();
+    final ILuceneClient client = new LuceneClient();
     final IQuery query = new Query(queryString);
     final long start = System.currentTimeMillis();
     final Hits hits = client.search(query, indexNames, count);
@@ -476,7 +500,7 @@
   }
 
   public static void search(final String[] indexNames, final String queryString) throws KattaException {
-    final IClient client = new Client();
+    final ILuceneClient client = new LuceneClient();
     final IQuery query = new Query(queryString);
     final long start = System.currentTimeMillis();
     final int hitsSize = client.count(query, indexNames);
@@ -499,21 +523,24 @@
     System.err.println("\tlistIndexes [-d]\tLists all indexes. -d for detailed view.");
     System.err.println("\tlistNodes\t\tLists all nodes.");
     System.err.println("\tstartMaster\t\tStarts a local master.");
-    System.err.println("\tstartNode\t\tStarts a local node.");
+    System.err.println("\tstartNode [server classname]\t\tStarts a local node.");
     System.err.println("\tshowStructure\t\tShows the structure of a Katta installation.");
     System.err.println("\tcheck\t\t\tAnalyze index/shard/node status.");
     System.err.println("\tversion\t\t\tPrint the version.");
     System.err
-        .println("\taddIndex <index name> <path to index> <lucene analyzer class> [<replication level>]\tAdd a index to a Katta installation.");
+        .println("\taddIndex <index name> <path to index> [<replication level>]\tAdd a index to a Katta installation.");
     System.err.println("\tremoveIndex <index name>\tRemove a index from a Katta installation.");
     System.err.println("\tredeployIndex <index name>\tUndeploys and deploys an index.");
     System.err
-        .println("\tmergeIndexes [-indexes <index1,index2>] [-hadoopSiteXml <siteXmlPath>]\tmergers all or the specified indexes.");
+        .println("\tmergeIndexes [-indexes <index1,index2>] [-hadoopSiteXml <siteXmlPath>]\tmerges all or the specified indexes.");
     System.err.println("\tlistErrors <index name>\t\tLists all deploy errors for a specified index.");
     System.err
-        .println("\tsearch <index name>[,<index name>,...] \"<query>\" [count]\tSearch in supplied indexes. The query should be in \". If you supply a result count hit details will be printed. To search in all indices write \"*\"");
+        .println("\tsearch <index name>[,<index name>,...] \"<query>\" [count]\tSearch in supplied indexes. " + 
+                 "The query should be in \". If you supply a result count hit details will be printed. " + 
+                 "To search in all indices write \"*\". This uses the client type LuceneClient.");
     System.err
-        .println("\tindex <inputTextFile> <outputPaht>  <numOfWordsPerDoc> <numOfDocuments> \tGenerates a sample index. The inputTextFile is used as dictionary.");
+        .println("\tindex <inputTextFile> <outputPaht>  <numOfWordsPerDoc> <numOfDocuments> \tGenerates a sample index. " + 
+                 "The inputTextFile is used as dictionary.");
     
     System.err.println();
     System.exit(1);
Index: src/main/java/net/sf/katta/node/KattaMultiSearcher.java
===================================================================
--- src/main/java/net/sf/katta/node/KattaMultiSearcher.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/main/java/net/sf/katta/node/KattaMultiSearcher.java	(.../trunk/katta)	(revision 10997)
@@ -1,397 +0,0 @@
-/**
- * Copyright 2008 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.sf.katta.node;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import org.apache.log4j.Logger;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DefaultSimilarity;
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.HitCollector;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Searchable;
-import org.apache.lucene.search.Searcher;
-import org.apache.lucene.search.Similarity;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.TopFieldDocs;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.util.PriorityQueue;
-
-/**
- * Implements search over a set of <code>Searchables</code>.
- * 
- * <p>
- * Applications usually need only call the inherited {@link #search(Query)} or
- * {@link #search(Query,Filter)} methods.
- */
-public class KattaMultiSearcher {
-
-  private final static Logger LOG = Logger.getLogger(KattaMultiSearcher.class);
-
-  private final Map<String, IndexSearcher> _searchers = new ConcurrentHashMap<String, IndexSearcher>();
-  private ExecutorService _threadPool = Executors.newFixedThreadPool(100);
-
-  private final String _node;
-  private int _maxDoc = 0;
-
-  public KattaMultiSearcher(final String node) {
-    _node = node;
-  }
-
-  /**
-   * Adds an shard index search for given name to the list of shards
-   * MultiSearcher search in.
-   * 
-   * @param shardKey
-   * @param indexSearcher
-   * @throws IOException
-   */
-  public void addShard(final String shardKey, final IndexSearcher indexSearcher) throws IOException {
-    synchronized (_searchers) {
-      _searchers.put(shardKey, indexSearcher);
-      _maxDoc += indexSearcher.maxDoc();
-    }
-  }
-
-  /**
-   * 
-   * Removes a search by given shardName from the list of searchers.
-   */
-  public void removeShard(final String shardName) {
-    synchronized (_searchers) {
-      final Searchable remove = _searchers.remove(shardName);
-      if (remove == null) {
-        return; // nothing to do.
-      }
-      try {
-        _maxDoc -= remove.maxDoc();
-      } catch (final IOException e) {
-        throw new RuntimeException("unable to retrive maxDocs from searchable");
-      }
-    }
-
-  }
-
-  /**
-   * Search in the given shards and return max hits for given query
-   * 
-   * @param query
-   * @param freqs
-   * @param shards
-   * @param result
-   * @param max
-   * @throws IOException
-   */
-  public final void search(final Query query, final DocumentFrequenceWritable freqs, final String[] shards,
-      final HitsMapWritable result, final int max) throws IOException {
-    final Query rewrittenQuery = rewrite(query, shards);
-    final int numDocs = freqs.getNumDocs();
-    final CachedDfSource cacheSim = new CachedDfSource(freqs.getAll(), numDocs, new DefaultSimilarity());
-    final Weight weight = rewrittenQuery.weight(cacheSim);
-    // we can maximal found all docs in this system or maximal the requested
-    final int limit = Math.min(numDocs, max);
-    final KattaHitQueue hq = new KattaHitQueue(limit);
-    int totalHits = 0;
-    final int shardsCount = shards.length;
-
-    // run the search parallel on the shards with a thread pool
-    List<Future<SearchResult>> tasks = new ArrayList<Future<SearchResult>>();
-    for (int i = 0; i < shardsCount; i++) {
-      SearchCall call = new SearchCall(shards[i], weight, limit);
-      Future<SearchResult> future = _threadPool.submit(call);
-      tasks.add(future);
-    }
-
-    final ScoreDoc[][] scoreDocs = new ScoreDoc[shardsCount][];
-    for (int i = 0; i < shardsCount; i++) {
-      SearchResult searchResult;
-      try {
-        searchResult = tasks.get(i).get();
-        totalHits += searchResult._totalHits;
-        scoreDocs[i] = searchResult._scoreDocs;
-      } catch (InterruptedException e) {
-        throw new IOException("Multithread shard search interrupred:", e);
-      } catch (ExecutionException e) {
-        throw new IOException("Multithread shard search could not be executed:", e);
-      }
-    }
-   
-    result.addTotalHits(totalHits);
-
-    int pos = 0;
-    boolean working = true;
-    while (working) {
-      ScoreDoc scoreDoc = null;
-      for (int i = 0; i < scoreDocs.length; i++) {
-        final ScoreDoc[] docs = scoreDocs[i];
-        if (pos < docs.length) {
-          scoreDoc = docs[pos];
-          final Hit hit = new Hit(shards[i], _node, scoreDoc.score, scoreDoc.doc);
-          if (!hq.insert(hit) || hq.size() == limit) {
-            working = false;
-            break;
-          }
-        }
-      }
-      pos++;
-      if (scoreDoc == null) {
-        // we do not have any data more
-        break;
-      }
-    }
-
-    for (int i = hq.size() - 1; i >= 0; i--) {
-      final Hit hit = (Hit) hq.pop();
-      if (hit != null) {
-        result.addHitToShard(hit.getShard(), hit);
-      }
-    }
-  }
-
-  /**
-   * Returns the number of documents a shard has.
-   * 
-   * @param shardName
-   * @return
-   */
-  public int getNumDoc(final String shardName) {
-    final Searchable searchable = _searchers.get(shardName);
-    if (searchable != null) {
-      final IndexSearcher indexSearcher = (IndexSearcher) searchable;
-      return indexSearcher.getIndexReader().numDocs();
-    }
-    throw new IllegalArgumentException("shard " + shardName + " unknown");
-  }
-
-  /**
-   * Returns the lucene document of a given shard.
-   * 
-   * @param shardName
-   * @param docId
-   * @return
-   * @throws CorruptIndexException
-   * @throws IOException
-   */
-  public Document doc(final String shardName, final int docId) throws CorruptIndexException, IOException {
-    final Searchable searchable = _searchers.get(shardName);
-    if (searchable != null) {
-      return searchable.doc(docId);
-    }
-    throw new IllegalArgumentException("shard " + shardName + " unknown");
-  }
-
-  /**
-   * Rewrites a query for the given shards
-   * 
-   * @param original
-   * @param shardNames
-   * @return
-   * @throws IOException
-   */
-  public Query rewrite(final Query original, final String[] shardNames) throws IOException {
-    final Query[] queries = new Query[shardNames.length];
-    for (int i = 0; i < shardNames.length; i++) {
-      final String shard = shardNames[i];
-      queries[i] = _searchers.get(shard).rewrite(original);
-    }
-    if (queries.length > 0) {
-      return queries[0].combine(queries);
-    }
-    return original;
-  }
-
-  /**
-   * Returns the document frequence for a given term within a given shard.
-   * 
-   * @param shardName
-   * @param term
-   * @return
-   * @throws IOException
-   */
-  public int docFreq(final String shardName, final Term term) throws IOException {
-    int result = 0;
-    final Searchable searchable = _searchers.get(shardName);
-    if (searchable != null) {
-      result = searchable.docFreq(term);
-    } else {
-      LOG.error("No shard with the name '" + shardName + "' on in this searcher.");
-    }
-    return result;
-  }
-
-  public void close() throws IOException {
-    for (final Searchable searchable : _searchers.values()) {
-      searchable.close();
-    }
-  }
-
-  private class SearchCall implements Callable<SearchResult> {
-
-    private final String _shardName;
-    private final Weight _weight;
-    private final int _limit;
-
-    public SearchCall(String shardName, Weight weight, int limit) {
-      _shardName = shardName;
-      _weight = weight;
-      _limit = limit;
-    }
-
-    @Override
-    public SearchResult call() throws Exception {
-      final IndexSearcher indexSearcher = _searchers.get(_shardName);
-      final TopDocs docs = indexSearcher.search(_weight, null, _limit);
-      // totalHits += docs.totalHits; // update totalHits
-      return new SearchResult(docs.totalHits, docs.scoreDocs);
-    }
-
-  }
-
-  private class SearchResult {
-
-    private final int _totalHits;
-    private final ScoreDoc[] _scoreDocs;
-
-    public SearchResult(int totalHits, ScoreDoc[] scoreDocs) {
-      _totalHits = totalHits;
-      _scoreDocs = scoreDocs;
-    }
-
-  }
-
-  // cached document frequence source from apache lucene
-  // MultiSearcher.
-  /**
-   * Document Frequency cache acting as a Dummy-Searcher. This class is no
-   * full-fledged Searcher, but only supports the methods necessary to
-   * initialize Weights.
-   */
-  private static class CachedDfSource extends Searcher {
-    private final Map dfMap; // Map from Terms to corresponding doc freqs
-
-    private final int maxDoc; // document count
-
-    public CachedDfSource(final Map dfMap, final int maxDoc, final Similarity similarity) {
-      this.dfMap = dfMap;
-      this.maxDoc = maxDoc;
-      setSimilarity(similarity);
-    }
-
-    @Override
-    public int docFreq(final Term term) {
-      int df;
-      try {
-        df = ((Integer) dfMap.get(new TermWritable(term.field(), term.text()))).intValue();
-      } catch (final NullPointerException e) {
-        throw new IllegalArgumentException("df for term " + term.text() + " not available");
-      }
-      return df;
-    }
-
-    @Override
-    public int[] docFreqs(final Term[] terms) {
-      final int[] result = new int[terms.length];
-      for (int i = 0; i < terms.length; i++) {
-        result[i] = docFreq(terms[i]);
-      }
-      return result;
-    }
-
-    @Override
-    public int maxDoc() {
-      return maxDoc;
-    }
-
-    @Override
-    public Query rewrite(final Query query) {
-      // this is a bit of a hack. We know that a query which
-      // creates a Weight based on this Dummy-Searcher is
-      // always already rewritten (see preparedWeight()).
-      // Therefore we just return the unmodified query here
-      return query;
-    }
-
-    @Override
-    public void close() {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Document doc(final int i) {
-      throw new UnsupportedOperationException();
-    }
-
-    public Document doc(final int i, final FieldSelector fieldSelector) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Explanation explain(final Weight weight, final int doc) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void search(final Weight weight, final Filter filter, final HitCollector results) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public TopDocs search(final Weight weight, final Filter filter, final int n) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public TopFieldDocs search(final Weight weight, final Filter filter, final int n, final Sort sort) {
-      throw new UnsupportedOperationException();
-    }
-  }
-
-  private class KattaHitQueue extends PriorityQueue {
-    KattaHitQueue(final int size) {
-      initialize(size);
-    }
-
-    @Override
-    protected final boolean lessThan(final Object a, final Object b) {
-      final Hit hitA = (Hit) a;
-      final Hit hitB = (Hit) b;
-      if (hitA.getScore() == hitB.getScore()) {
-        // todo this of cource do not work since we have same shardKeys
-        // (should we increment docIds?)
-        return hitA.getDocId() > hitB.getDocId();
-      }
-      return hitA.getScore() < hitB.getScore();
-    }
-  }
-
-}
Index: src/main/java/net/sf/katta/node/ISearch.java
===================================================================
--- src/main/java/net/sf/katta/node/ISearch.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/main/java/net/sf/katta/node/ISearch.java	(.../trunk/katta)	(revision 10997)
@@ -1,104 +0,0 @@
-/**
- * Copyright 2008 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.sf.katta.node;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.MapWritable;
-import org.apache.hadoop.ipc.VersionedProtocol;
-import org.apache.lucene.queryParser.ParseException;
-
-public interface ISearch extends VersionedProtocol {
-
-  /**
-   * Returns all Hits that match the query. This might be significant slower as
-   * {@link #search(IQuery, DocumentFrequenceWritable, String[], int)} since we
-   * replace count with {@link Integer.MAX_VALUE}.
-   * 
-   * @param query
-   * @param freqs
-   * @param shardNames
-   *          A array of shard names to search in.
-   * @return
-   * @throws ParseException
-   * @throws IOException
-   */
-  public HitsMapWritable search(QueryWritable query, DocumentFrequenceWritable freqs, String[] shardNames) throws IOException;
-
-  /**
-   * @param query
-   * @param freqs
-   * @param shardNames
-   * @param count
-   *          the top n high score hits
-   * @return
-   * @throws ParseException
-   * @throws IOException
-   */
-  public HitsMapWritable search(QueryWritable query, DocumentFrequenceWritable freqs, String[] shardNames, int count)
-      throws IOException;
-
-  /**
-   * Returns the number of documents a term occurs in. In a distributed search
-   * environment, we need to get this first and then query all nodes again with
-   * this information to ensure we compute TF IDF correctly. See {@link http
-   * ://lucene
-   * .apache.org/java/2_3_0/api/org/apache/lucene/search/Similarity.html}
-   * 
-   * @param input
-   * @param shards
-   * @return
-   * @throws IOException
-   * @throws ParseException
-   */
-  public DocumentFrequenceWritable getDocFreqs(QueryWritable input, String[] shards) throws IOException;
-
-  /**
-   * Returns only the request fields of a lucene document.
-   * 
-   * @param shard
-   * @param docId
-   * @param fields
-   * @return
-   * @throws IOException
-   */
-  public MapWritable getDetails(String shard, int docId, String[] fields) throws IOException;
-
-  /**
-   * Returns the lucene document. Each field:value tuple of the lucene document
-   * is pushed ito the map. In most cases
-   * {@link #getDetails(String, int, String[])} would be a better choice for
-   * performance reasons.
-   * 
-   * @param shard
-   * @param docId
-   * @return
-   * @throws IOException
-   */
-  public MapWritable getDetails(String shard, int docId) throws IOException;
-
-  /**
-   * Returns the number of documents that match the given query. This the
-   * fastest way in case you just need the number of documents. Note that the
-   * number of matching documents is also included in HitsMapWritable.
-   * 
-   * @param query
-   * @param strings
-   * @return
-   * @throws IOException
-   */
-  public int getResultCount(QueryWritable query, String[] strings) throws IOException;
-}
Index: src/main/java/net/sf/katta/node/LuceneServer.java
===================================================================
--- src/main/java/net/sf/katta/node/LuceneServer.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/main/java/net/sf/katta/node/LuceneServer.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,554 @@
+/**
+ * Copyright 2008 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.node;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.HitCollector;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Searchable;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.PriorityQueue;
+
+/**
+ * The back end server which searches a set of Lucene indices.
+ * Each shard is a Lucene index directory.
+ * <p>
+ * Normal usage is to first call getDocFreqs() to get the global
+ * term frequencies, then pass that back in to search(). This way
+ * you get uniform scoring across all the nodes / instances of 
+ * LuceneServer.
+ */
+public class LuceneServer implements INodeManaged, ILuceneServer {
+
+  private final static Logger LOG = Logger.getLogger(LuceneServer.class);
+
+  private final Map<String, IndexSearcher> _searchers = new ConcurrentHashMap<String, IndexSearcher>();
+  private ExecutorService _threadPool = Executors.newFixedThreadPool(100);
+
+  private String _nodeName;
+  private int _maxDoc = 0;
+
+  public LuceneServer() {
+  }
+
+  public long getProtocolVersion(final String protocol, final long clientVersion) throws IOException {
+    return 0L;
+  }
+
+  public void setNodeName(String nodeName) {
+    _nodeName = nodeName;
+  }
+
+  /**
+   * Adds an shard index search for given name to the list of shards
+   * MultiSearcher search in.
+   * 
+   * @param shardName
+   * @param indexSearcher
+   * @throws IOException
+   */
+  public void addShard(final String shardName, final File shardDir) throws IOException {
+    LOG.debug("LuceneServer " + _nodeName + " got shard " + shardName);
+    try {
+      IndexSearcher indexSearcher = new IndexSearcher(shardDir.getAbsolutePath());
+      synchronized (_searchers) {
+        _searchers.put(shardName, indexSearcher);
+        _maxDoc += indexSearcher.maxDoc();
+      }
+    } catch (CorruptIndexException e) {
+      LOG.error("Error building index for shard " + shardName, e);
+      throw e;
+    }
+  }
+
+  /**
+   * 
+   * Removes a search by given shardName from the list of searchers.
+   */
+  public void removeShard(final String shardName) {
+    LOG.debug("LuceneServer " + _nodeName + " removing shard " + shardName);
+    synchronized (_searchers) {
+      final Searchable remove = _searchers.remove(shardName);
+      if (remove == null) {
+        return; // nothing to do.
+      }
+      try {
+        _maxDoc -= remove.maxDoc();
+      } catch (final IOException e) {
+        throw new RuntimeException("unable to retrive maxDocs from searchable");
+      }
+    }
+  }
+
+  /**
+   * Returns the number of documents a shard has.
+   * 
+   * @param shardName
+   * @return the number of documents in the shard.
+   */
+  public int shardSize(final String shardName) {
+    final Searchable searchable = _searchers.get(shardName);
+    if (searchable != null) {
+      final IndexSearcher indexSearcher = (IndexSearcher) searchable;
+      return indexSearcher.getIndexReader().numDocs();
+    }
+    throw new IllegalArgumentException("shard " + shardName + " unknown");
+  }
+
+  /**
+   * Close all Lucene indices. No further calls will be made after this one.
+   */
+  public void shutdown() throws IOException {
+    for (final Searchable searchable : _searchers.values()) {
+      searchable.close();
+    }
+    _searchers.clear();
+  }
+
+
+
+
+  public HitsMapWritable search(final QueryWritable query, final DocumentFrequenceWritable freqs, final String[] shards,
+          final int count) throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("You are searching with the query: '" + query.getQuery() + "'");
+    }
+
+    Query luceneQuery = query.getQuery();
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Lucene query: " + luceneQuery.toString());
+    }
+
+    long completeSearchTime = 0;
+    final HitsMapWritable result = new net.sf.katta.node.HitsMapWritable(_nodeName);
+    long start = 0;
+    if (LOG.isDebugEnabled()) {
+      start = System.currentTimeMillis();
+    }
+    search(luceneQuery, freqs, shards, result, count);
+    if (LOG.isDebugEnabled()) {
+      final long end = System.currentTimeMillis();
+      LOG.debug("Search took " + (end - start) / 1000.0 + "sec.");
+      completeSearchTime += (end - start);
+    }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Complete search took " + completeSearchTime / 1000.0 + "sec.");
+      final DataOutputBuffer buffer = new DataOutputBuffer();
+      result.write(buffer);
+      LOG.debug("Result size to transfer: " + buffer.getLength());
+    }
+    return result;
+  }
+
+
+  public DocumentFrequenceWritable getDocFreqs(final QueryWritable input, final String[] shards) throws IOException {
+    Query luceneQuery = input.getQuery();
+
+    final Query rewrittenQuery = rewrite(luceneQuery, shards);
+    final DocumentFrequenceWritable docFreqs = new DocumentFrequenceWritable();
+
+    final HashSet<Term> termSet = new HashSet<Term>();
+    rewrittenQuery.extractTerms(termSet);
+    int numDocs = 0;
+    for (final String shard : shards) {
+      final java.util.Iterator<Term> termIterator = termSet.iterator();
+      while (termIterator.hasNext()) {
+        final Term term = termIterator.next();
+        final int docFreq = docFreq(shard, term);
+        docFreqs.put(term.field(), term.text(), docFreq);
+      }
+      numDocs += shardSize(shard);
+    }
+    docFreqs.setNumDocs(numDocs);
+    return docFreqs;
+  }
+
+
+  @SuppressWarnings("unchecked")
+  public MapWritable getDetails(final String[] shards, final int docId) throws IOException {
+    final MapWritable result = new MapWritable();
+    final Document doc = doc(shards[0], docId);
+    final List<Fieldable> fields = doc.getFields();
+    for (final Fieldable field : fields) {
+      final String name = field.name();
+      if (field.isBinary()) {
+        final byte[] binaryValue = field.binaryValue();
+        result.put(new Text(name), new BytesWritable(binaryValue));
+      } else {
+        final String stringValue = field.stringValue();
+        result.put(new Text(name), new Text(stringValue));
+      }
+    }
+    return result;
+  }
+
+  @SuppressWarnings("deprecation")
+  public MapWritable getDetails(final String[] shards, final int docId, final String[] fieldNames) throws IOException {
+    final MapWritable result = new MapWritable();
+    final Document doc = doc(shards[0], docId);
+    for (final String fieldName : fieldNames) {
+      final Field field = doc.getField(fieldName);
+      if (field != null) {
+        if (field.isBinary()) {
+          final byte[] binaryValue = field.binaryValue();
+          result.put(new Text(fieldName), new BytesWritable(binaryValue));
+        } else {
+          final String stringValue = field.stringValue();
+          result.put(new Text(fieldName), new Text(stringValue));
+        }
+      }
+    }
+    return result;
+  }
+
+  public int getResultCount(final QueryWritable query, final String[] shards) throws IOException {
+    final DocumentFrequenceWritable docFreqs = getDocFreqs(query, shards);
+    return search(query, docFreqs, shards, 1).getTotalHits();
+  }
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  /**
+   * Search in the given shards and return max hits for given query
+   * 
+   * @param query
+   * @param freqs
+   * @param shards
+   * @param result
+   * @param max
+   * @throws IOException
+   */
+  private final void search(final Query query, final DocumentFrequenceWritable freqs, final String[] shards,
+          final HitsMapWritable result, final int max) throws IOException {
+    final Query rewrittenQuery = rewrite(query, shards);
+    final int numDocs = freqs.getNumDocs();
+    final CachedDfSource cacheSim = new CachedDfSource(freqs.getAll(), numDocs, new DefaultSimilarity());
+    final Weight weight = rewrittenQuery.weight(cacheSim);
+    // we can maximal found all docs in this system or maximal the requested
+    final int limit = Math.min(numDocs, max);
+    final KattaHitQueue hq = new KattaHitQueue(limit);
+    int totalHits = 0;
+    final int shardsCount = shards.length;
+
+    // run the search parallel on the shards with a thread pool
+    List<Future<SearchResult>> tasks = new ArrayList<Future<SearchResult>>();
+    for (int i = 0; i < shardsCount; i++) {
+      SearchCall call = new SearchCall(shards[i], weight, limit);
+      Future<SearchResult> future = _threadPool.submit(call);
+      tasks.add(future);
+    }
+
+    final ScoreDoc[][] scoreDocs = new ScoreDoc[shardsCount][];
+    for (int i = 0; i < shardsCount; i++) {
+      SearchResult searchResult;
+      try {
+        searchResult = tasks.get(i).get();
+        totalHits += searchResult._totalHits;
+        scoreDocs[i] = searchResult._scoreDocs;
+      } catch (InterruptedException e) {
+        throw new IOException("Multithread shard search interrupred:", e);
+      } catch (ExecutionException e) {
+        throw new IOException("Multithread shard search could not be executed:", e);
+      }
+    }
+
+    result.addTotalHits(totalHits);
+
+    int pos = 0;
+    boolean working = true;
+    while (working) {
+      ScoreDoc scoreDoc = null;
+      for (int i = 0; i < scoreDocs.length; i++) {
+        final ScoreDoc[] docs = scoreDocs[i];
+        if (pos < docs.length) {
+          scoreDoc = docs[pos];
+          final Hit hit = new Hit(shards[i], _nodeName, scoreDoc.score, scoreDoc.doc);
+          if (!hq.insert(hit) || hq.size() == limit) {
+            working = false;
+            break;
+          }
+        }
+      }
+      pos++;
+      if (scoreDoc == null) {
+        // we do not have any data more
+        break;
+      }
+    }
+
+    for (int i = hq.size() - 1; i >= 0; i--) {
+      final Hit hit = (Hit) hq.pop();
+      if (hit != null) {
+        result.addHitToShard(hit.getShard(), hit);
+      }
+    }
+  }
+
+  /**
+   * Returns the lucene document of a given shard.
+   * 
+   * @param shardName
+   * @param docId
+   * @return
+   * @throws CorruptIndexException
+   * @throws IOException
+   */
+  private Document doc(final String shardName, final int docId) throws CorruptIndexException, IOException {
+    final Searchable searchable = _searchers.get(shardName);
+    if (searchable != null) {
+      return searchable.doc(docId);
+    }
+    throw new IllegalArgumentException("shard " + shardName + " unknown");
+  }
+
+  /**
+   * Rewrites a query for the given shards
+   * 
+   * @param original
+   * @param shardNames
+   * @return
+   * @throws IOException
+   */
+  private Query rewrite(final Query original, final String[] shardNames) throws IOException {
+    final Query[] queries = new Query[shardNames.length];
+    for (int i = 0; i < shardNames.length; i++) {
+      final String shard = shardNames[i];
+      final IndexSearcher searcher = _searchers.get(shard);
+      if (searcher == null) {
+        LOG.error("Node " + _nodeName + ": unknown shard " + shard);
+      }
+      queries[i] = searcher.rewrite(original);
+    }
+    if (queries.length > 0) {
+      return queries[0].combine(queries);
+    }
+    return original;
+  }
+
+  /**
+   * Returns the document frequence for a given term within a given shard.
+   * 
+   * @param shardName
+   * @param term
+   * @return
+   * @throws IOException
+   */
+  private int docFreq(final String shardName, final Term term) throws IOException {
+    int result = 0;
+    final Searchable searchable = _searchers.get(shardName);
+    if (searchable != null) {
+      result = searchable.docFreq(term);
+    } else {
+      LOG.error("No shard with the name '" + shardName + "' on in this searcher.");
+    }
+    return result;
+  }
+
+//  private void close() throws IOException {
+//    for (final Searchable searchable : _searchers.values()) {
+//      searchable.close();
+//    }
+//  }
+
+  private class SearchCall implements Callable<SearchResult> {
+
+    private final String _shardName;
+    private final Weight _weight;
+    private final int _limit;
+
+    public SearchCall(String shardName, Weight weight, int limit) {
+      _shardName = shardName;
+      _weight = weight;
+      _limit = limit;
+    }
+
+    @Override
+    public SearchResult call() throws Exception {
+      final IndexSearcher indexSearcher = _searchers.get(_shardName);
+      final TopDocs docs = indexSearcher.search(_weight, null, _limit);
+      // totalHits += docs.totalHits; // update totalHits
+      return new SearchResult(docs.totalHits, docs.scoreDocs);
+    }
+
+  }
+
+  private class SearchResult {
+
+    private final int _totalHits;
+    private final ScoreDoc[] _scoreDocs;
+
+    public SearchResult(int totalHits, ScoreDoc[] scoreDocs) {
+      _totalHits = totalHits;
+      _scoreDocs = scoreDocs;
+    }
+
+  }
+
+  // cached document frequence source from apache lucene
+  // MultiSearcher.
+  /**
+   * Document Frequency cache acting as a Dummy-Searcher. This class is no
+   * full-fledged Searcher, but only supports the methods necessary to
+   * initialize Weights.
+   */
+  private static class CachedDfSource extends Searcher {
+    @SuppressWarnings("unchecked")
+    private final Map dfMap; // Map from Terms to corresponding doc freqs
+
+    private final int maxDoc; // document count
+
+    @SuppressWarnings("unchecked")
+    public CachedDfSource(final Map dfMap, final int maxDoc, final Similarity similarity) {
+      this.dfMap = dfMap;
+      this.maxDoc = maxDoc;
+      setSimilarity(similarity);
+    }
+
+    @Override
+    public int docFreq(final Term term) {
+      int df;
+      try {
+        df = ((Integer) dfMap.get(new TermWritable(term.field(), term.text()))).intValue();
+      } catch (final NullPointerException e) {
+        throw new IllegalArgumentException("df for term " + term.text() + " not available");
+      }
+      return df;
+    }
+
+    @Override
+    public int[] docFreqs(final Term[] terms) {
+      final int[] result = new int[terms.length];
+      for (int i = 0; i < terms.length; i++) {
+        result[i] = docFreq(terms[i]);
+      }
+      return result;
+    }
+
+    @Override
+    public int maxDoc() {
+      return maxDoc;
+    }
+
+    @Override
+    public Query rewrite(final Query query) {
+      // this is a bit of a hack. We know that a query which
+      // creates a Weight based on this Dummy-Searcher is
+      // always already rewritten (see preparedWeight()).
+      // Therefore we just return the unmodified query here
+      return query;
+    }
+
+    @Override
+    public void close() {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Document doc(final int i) {
+      throw new UnsupportedOperationException();
+    }
+
+    public Document doc(final int i, final FieldSelector fieldSelector) {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Explanation explain(final Weight weight, final int doc) {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void search(final Weight weight, final Filter filter, final HitCollector results) {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public TopDocs search(final Weight weight, final Filter filter, final int n) {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public TopFieldDocs search(final Weight weight, final Filter filter, final int n, final Sort sort) {
+      throw new UnsupportedOperationException();
+    }
+  }
+
+  private class KattaHitQueue extends PriorityQueue {
+    KattaHitQueue(final int size) {
+      initialize(size);
+    }
+
+    @Override
+    protected final boolean lessThan(final Object a, final Object b) {
+      final Hit hitA = (Hit) a;
+      final Hit hitB = (Hit) b;
+      if (hitA.getScore() == hitB.getScore()) {
+        // todo this of cource do not work since we have same shardKeys
+        // (should we increment docIds?)
+        return hitA.getDocId() > hitB.getDocId();
+      }
+      return hitA.getScore() < hitB.getScore();
+    }
+  }
+
+}
Index: src/main/java/net/sf/katta/node/MapFileServer.java
===================================================================
--- src/main/java/net/sf/katta/node/MapFileServer.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/main/java/net/sf/katta/node/MapFileServer.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,234 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.node;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.MapFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.MapFile.Reader;
+import org.apache.log4j.Logger;
+
+/**
+ * Implements search over a set of Hadoop <code>MapFile</code>s.
+ */
+public class MapFileServer implements INodeManaged, IMapFileServer {
+
+  private final static Logger LOG = Logger.getLogger(MapFileServer.class);
+
+  private final Configuration _conf = new Configuration();
+  private final FileSystem _fileSystem = new RawLocalFileSystem();
+  private final Map<String, MapFile.Reader> _readers = new ConcurrentHashMap<String, MapFile.Reader>();
+  private String _nodeName;
+
+  public MapFileServer() {
+    _fileSystem.setConf(_conf);
+  }
+
+  public long getProtocolVersion(final String protocol, final long clientVersion) throws IOException {
+    return 0L;
+  }
+
+  public void setNodeName(String nodeName) {
+    _nodeName = nodeName;
+  }
+
+  /**
+   * Adds an shard index search for given name to the list of shards
+   * MultiSearcher search in.
+   * 
+   * @param shardName
+   * @param indexSearcher
+   * @throws IOException
+   */
+  public void addShard(final String shardName, final File shardDir) throws IOException {
+    LOG.debug("LuceneServer " + _nodeName + " got shard " + shardName);
+    if (!shardDir.exists()) {
+      throw new IOException("Shard " + shardName + " dir " + shardDir.getAbsolutePath() + " does not exist!");
+    }
+    if (!shardDir.canRead()) {
+      throw new IOException("Can not read shard " + shardName + " dir " + shardDir.getAbsolutePath() + "!");
+    }
+    try {
+      final MapFile.Reader reader = new MapFile.Reader(_fileSystem, shardDir.getAbsolutePath(), _conf);
+      synchronized (_readers) {
+        _readers.put(shardName, reader);
+      }
+    } catch (IOException e) {
+      LOG.error("Error opening shard " + shardName + " " + shardDir.getAbsolutePath(), e);
+      throw e;
+    }
+  }
+
+  /**
+   * 
+   * Removes a search by given shardName from the list of searchers.
+   */
+  public void removeShard(final String shardName) throws IOException {
+    LOG.debug("LuceneServer " + _nodeName + " removing shard " + shardName);
+    synchronized (_readers) {
+      final MapFile.Reader reader = _readers.get(shardName);
+      if (reader != null) {
+        try {
+          reader.close();
+        } catch (IOException e) {
+          LOG.error("Error closing shard " + shardName, e);
+          throw e;
+        }
+        _readers.remove(shardName);
+      } else {
+        LOG.warn("Shard " + shardName + " not found!");
+      }
+    }
+  }
+  
+  /**
+   * Returns the number of entries this MapFile has.
+   * 
+   * @param shardName The shard to measure.
+   * @return the number of entries in this MapFile.
+   */
+  public int shardSize(final String shardName) throws Exception {
+    final MapFile.Reader reader = _readers.get(shardName);
+    if (reader != null) {
+      int count = 0;
+      synchronized (reader) {
+        reader.reset();
+        WritableComparable<?> key = (WritableComparable<?>) reader.getKeyClass().newInstance();
+        Writable value = (Writable) reader.getValueClass().newInstance();
+        while (reader.next(key, value)) {
+          count++;
+        }
+      }
+      return count;
+    } else {
+      LOG.warn("Shard " + shardName + " not found!");
+      throw new IllegalArgumentException("Shard " + shardName + " unknown");
+    }
+  }
+
+  /**
+   * Close all MapFiles. No further calls will be made after this one.
+   */
+  public void shutdown() throws IOException {
+    for (final MapFile.Reader reader : _readers.values()) {
+      try {
+        reader.close();
+      } catch (IOException e) {
+        LOG.error("Error in shutdown", e);
+      }
+    }
+    _readers.clear();
+  }
+
+
+  public TextArrayWritable get(Text key, String[] shards) throws IOException {
+    ExecutorService executor = Executors.newCachedThreadPool();
+    Collection<Future<Text>> futures = new ArrayList<Future<Text>>();
+    for (String shard : shards) {
+      final MapFile.Reader reader = _readers.get(shard);
+      if (reader == null) {
+        LOG.warn("Shard " + shard + " unknown");
+        continue;
+      }
+      Callable<Text> callable = new MapLookup(reader, key);
+      futures.add(executor.submit(callable));
+    }
+    executor.shutdown();
+    try {
+      executor.awaitTermination(1, TimeUnit.MINUTES); // TODO: config, 10 sec?
+    } catch (InterruptedException e) {
+      LOG.warn("Interrupted while waiting on MapLookup threads", e);
+    }
+    executor.shutdownNow();
+    List<Text> resultList = new ArrayList<Text>();
+    for (Future<Text> future : futures) {
+      try {
+        Text result = future.get(0, TimeUnit.MILLISECONDS);
+        if (result != null) {
+          resultList.add(result);
+        }
+      } catch (ExecutionException e) {
+        /*
+         *  This MapFile red threw an exception.
+         *  Stop processing and throw an IOE.
+         */
+        Throwable t = e.getCause();
+        if (t instanceof IOException) {
+          // Throw the same IOException that the MapFile.Reader threw.
+          throw (IOException) t;
+        } else {
+          // Wrap MapFile.Reader's exception in an IOException.
+          throw new IOException("Error in MapLookup", t);
+        }
+      } catch (TimeoutException e) {
+        /*
+         * Result is not ready. Should not happen, because future is done.
+         * Continue as if MapLookup had returned null.
+         */
+        LOG.warn("Timed out while getting MapLookup", e);
+      } catch (InterruptedException e) {
+        /*
+         *  Something went wrong while waiting for result. Should not happen
+         *  because we wait for 0 msec, and the future is done. Continue as if
+         *  the MapLookup had returned null.
+         */
+        LOG.warn("Interrupted while getting RPC result", e);
+      }
+    }
+    return new TextArrayWritable(resultList);
+  }
+  
+  
+  private class MapLookup implements Callable<Text> {
+
+    private MapFile.Reader _reader;
+    private WritableComparable<?> _key;
+    
+    public MapLookup(Reader reader, WritableComparable<?> key) {
+      _reader = reader;
+      _key = key;
+    }
+
+    public Text call() throws Exception {
+      synchronized (_reader) {
+        Writable result = (Writable) _reader.getValueClass().newInstance();
+        result = _reader.get(_key, result);
+        return (Text) result;
+      }
+    }
+    
+  }
+
+}
Index: src/main/java/net/sf/katta/node/Hits.java
===================================================================
--- src/main/java/net/sf/katta/node/Hits.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/main/java/net/sf/katta/node/Hits.java	(.../trunk/katta)	(revision 10997)
@@ -96,6 +96,7 @@
     sortCollection(count);
   }
 
+  @SuppressWarnings("unchecked")
   public void sortMerge() {
     final List<Hit>[] array = _hitsList.toArray(new List[_hitsList.size()]);
     _hitsList = new ArrayList<List<Hit>>();
@@ -186,6 +187,12 @@
 
   @Override
   public String toString() {
-    return getHits().toString();
+    /*
+     * Don't modify data structure just by viewing it, otherwise
+     * running in a debugger modifies the behavior of the code!
+     */
+    return "Hits: total=" + _totalHits + ", queue=" + (_hitsList != null ? _hitsList.toString() : "null") +
+      ", sorted=" + (_sortedList != null ? _sortedList.toString() : "null");
   }
+  
 }
Index: src/main/java/net/sf/katta/node/ILuceneServer.java
===================================================================
--- src/main/java/net/sf/katta/node/ILuceneServer.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/main/java/net/sf/katta/node/ILuceneServer.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,93 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.node;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.lucene.queryParser.ParseException;
+
+/**
+ * The public interface to the back end LuceneServer. These are all the
+ * methods that the Hadoop RPC will call.
+ */
+public interface ILuceneServer extends VersionedProtocol {
+
+  /**
+   * @param query
+   * @param freqs
+   * @param shardNames
+   * @param count
+   *          the top n high score hits
+   * @return
+   * @throws ParseException
+   * @throws IOException
+   */
+  public HitsMapWritable search(QueryWritable query, DocumentFrequenceWritable freqs, String[] shardNames, int count)
+      throws IOException;
+
+  /**
+   * Returns the number of documents a term occurs in. In a distributed search
+   * environment, we need to get this first and then query all nodes again with
+   * this information to ensure we compute TF IDF correctly. See {@link http
+   * ://lucene
+   * .apache.org/java/2_3_0/api/org/apache/lucene/search/Similarity.html}
+   * 
+   * @param input
+   * @param shards
+   * @return
+   * @throws IOException
+   * @throws ParseException
+   */
+  public DocumentFrequenceWritable getDocFreqs(QueryWritable input, String[] shards) throws IOException;
+
+  /**
+   * Returns only the request fields of a lucene document.
+   * 
+   * @param shards
+   * @param docId
+   * @param fields
+   * @return
+   * @throws IOException
+   */
+  public MapWritable getDetails(String[] shards, int docId, String[] fields) throws IOException;
+
+  /**
+   * Returns the lucene document. Each field:value tuple of the lucene document
+   * is pushed ito the map. In most cases
+   * {@link #getDetails(String, int, String[])} would be a better choice for
+   * performance reasons.
+   * 
+   * @param shards
+   * @param docId
+   * @return
+   * @throws IOException
+   */
+  public MapWritable getDetails(String[] shards, int docId) throws IOException;
+
+  /**
+   * Returns the number of documents that match the given query. This the
+   * fastest way in case you just need the number of documents. Note that the
+   * number of matching documents is also included in HitsMapWritable.
+   * 
+   * @param query
+   * @param shards
+   * @return
+   * @throws IOException
+   */
+  public int getResultCount(QueryWritable query, String[] shards) throws IOException;
+}
Index: src/main/java/net/sf/katta/node/INodeManaged.java
===================================================================
--- src/main/java/net/sf/katta/node/INodeManaged.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/main/java/net/sf/katta/node/INodeManaged.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,75 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.node;
+
+import java.io.File;
+
+/**
+ * This describes the interaction between the general Node class and 
+ * a specific Katta server instance it is managing. The Node class
+ * talks to Zookeeper, and manages the shards on disk. It tells the
+ * server when to start and stop using the shards, and when to shut down.
+ * 
+ * The RPC calls from the client will not go through the Node.
+ * The Hadoop RPC.Server uses a separate interface for those calls.
+ */
+public interface INodeManaged {
+
+  /**
+   * The name of the local machine, for example "sever21.foo.com:8000".
+   * Use this name in your results if you need to refer to the current node.
+   * 
+   * @param nodeName the identifier for the current node.
+   */
+  public void setNodeName(String nodeName);
+  
+  /**
+   * Include the shard (directory of data) when computing results.
+   * The shard is a directory, ready to be used.
+   *  
+   * @param shardName The name of the shard. Will be used in 
+   * removeShard(). May also be used in requests.
+   * @param shardDir The directory where the shard data is.
+   * @throws Exception 
+   */
+  public void addShard(String shardName, File shardDir) throws Exception;
+  
+  /**
+   * Stop including the shard (directory of data). After
+   * this call returns, the server should use the directory,
+   * or even assume that it exists.
+   * 
+   * @param shardName Which shard to stop using. This was the name
+   * provided in addShard().
+   */
+  public void removeShard(String shardName) throws Exception;
+  
+  /**
+   * Returns the "size" of a shard, using whatever units the server chooses.
+   * 
+   * @param shardName The name of the shard to measure. 
+   * This was the name provided in addShard().
+   * @return an integer describing the size of the shard.
+   * @throws Exception 
+   */
+  public int shardSize(String shardName) throws Exception;
+  
+  /**
+   * Release all resources. No further calls will happen after this call.
+   */
+  public void shutdown() throws Exception;
+  
+}
Index: src/main/java/net/sf/katta/node/TextArrayWritable.java
===================================================================
--- src/main/java/net/sf/katta/node/TextArrayWritable.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/main/java/net/sf/katta/node/TextArrayWritable.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,54 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.node;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.io.ArrayWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This class provides a way to return a list of Text objects via
+ * Hadoop RPC. It provides a zero-arg constructor, which Hadoop RPC
+ * requires for return types.
+ */
+public class TextArrayWritable implements Writable {
+
+  public ArrayWritable array;
+  
+  @SuppressWarnings("unchecked")
+  public TextArrayWritable() {
+    this((List<Text>) Collections.EMPTY_LIST);
+  }
+  
+  public TextArrayWritable(List<Text> texts) {
+    array = new ArrayWritable(Text.class, texts.toArray(new Writable[texts.size()]));
+  }
+  
+  public void readFields(DataInput in) throws IOException {
+    array.readFields(in);
+  }
+
+  public void write(DataOutput out) throws IOException {
+    array.write(out);
+  }
+  
+}
Index: src/main/java/net/sf/katta/node/IMapFileServer.java
===================================================================
--- src/main/java/net/sf/katta/node/IMapFileServer.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/main/java/net/sf/katta/node/IMapFileServer.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,41 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.node;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.VersionedProtocol;
+
+/**
+ * Interface for the client calls that will arrive via Hadoop RPC.
+ * 
+ * This server looks up Text entries from MapFiles using Text keys.
+ */
+public interface IMapFileServer extends VersionedProtocol {
+
+  /**
+   * Get all the occurrences of the given Text key. There could be
+   * up to one entry per shard.
+   * 
+   * @param key The key to search for.
+   * @param shards Which MapFile shards to look in.
+   * @return The list of Text results.
+   * @throws IOException If an error occurs.
+   */
+  public TextArrayWritable get(Text key, String[] shards) throws IOException;
+  
+}
Index: src/main/java/net/sf/katta/node/Node.java
===================================================================
--- src/main/java/net/sf/katta/node/Node.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/main/java/net/sf/katta/node/Node.java	(.../trunk/katta)	(revision 10997)
@@ -47,25 +47,11 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.MapWritable;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RPC.Server;
 import org.apache.log4j.Logger;
-import org.apache.lucene.analysis.KeywordAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
 
-public class Node implements ISearch, IZkReconnectListener {
+public class Node implements IZkReconnectListener {
 
   protected final static Logger LOG = Logger.getLogger(Node.class);
 
@@ -73,7 +59,7 @@
 
   protected ZKClient _zkClient;
   private Server _rpcServer;
-  private KattaMultiSearcher _searcher;
+  private INodeManaged _server;
 
   protected String _nodeName;
   protected int _searchServerPort;
@@ -92,13 +78,17 @@
     STARTING, RECONNECTING, IN_SERVICE, LOST;
   }
 
-  public Node(final ZKClient zkClient) {
-    this(zkClient, new NodeConfiguration());
+  public Node(final ZKClient zkClient, INodeManaged server) {
+    this(zkClient, new NodeConfiguration(), server);
   }
 
-  public Node(final ZKClient zkClient, final NodeConfiguration configuration) {
+  public Node(final ZKClient zkClient, final NodeConfiguration configuration, INodeManaged server) {
+    if (server == null) {
+      throw new IllegalArgumentException("Null server passed to Node()");
+    }
     _zkClient = zkClient;
     _configuration = configuration;
+    _server = server;
     _zkClient.subscribeReconnects(this);
   }
 
@@ -114,6 +104,7 @@
       _zkClient.getEventLock().lock();
       LOG.debug("Starting rpc search server...");
       _nodeName = startRPCServer(_configuration);
+      _server.setNodeName(_nodeName);
 
       // we add hostName and port to the shardFolder to allow multiple nodes per
       // server with the same configuration
@@ -152,7 +143,7 @@
 
   private void cleanupLocalShardFolder() throws KattaException {
     String node2ShardRootPath = ZkPathes.getNode2ShardRootPath(_nodeName);
-    List<String> shardsToServe = Collections.EMPTY_LIST;
+    List<String> shardsToServe = Collections.emptyList();
     if (_zkClient.exists(node2ShardRootPath)) {
       shardsToServe = _zkClient.getChildren(node2ShardRootPath);
     }
@@ -213,11 +204,11 @@
         if (!localShardFolder.exists()) {
           installShard(shard, localShardFolder);
         }
-        serveShard(shardName, localShardFolder);
+        _server.addShard(shardName, localShardFolder);
         announceShard(shard);
-      } catch (Exception e) {
-        LOG.error(_nodeName + ": could not deploy shard '" + shard + "'", e);
-        ShardError shardError = new ShardError(e.getMessage());
+      } catch (Throwable t) {
+        LOG.error(_nodeName + ": could not deploy shard '" + shard + "'", t);
+        ShardError shardError = new ShardError(t.getMessage());
         String shard2ErrorPath = ZkPathes.getShard2ErrorPath(shardName, _nodeName);
         if (_zkClient.exists(shard2ErrorPath)) {
           LOG.warn("detected old shard-to-error entry - deleting it..");
@@ -234,7 +225,7 @@
     for (String shard : shardsToRemove) {
       try {
         LOG.info("Undeploying shard: " + shard);
-        _searcher.removeShard(shard);
+        _server.removeShard(shard);
         String shard2NodePath = ZkPathes.getShard2NodePath(shard, _nodeName);
         if (_zkClient.exists(shard2NodePath)) {
           _zkClient.delete(shard2NodePath);
@@ -247,15 +238,6 @@
   }
 
   /*
-   * Creates an index search and adds it to the KattaMultiSearch
-   */
-  private void serveShard(final String shardName, final File localShardFolder) throws CorruptIndexException,
-      IOException {
-    IndexSearcher indexSearcher = new IndexSearcher(localShardFolder.getAbsolutePath());
-    _searcher.addShard(shardName, indexSearcher);
-  }
-
-  /*
    * Announce in zookeeper node is serving this shard,
    */
   private void announceShard(AssignedShard shard) throws KattaException {
@@ -269,7 +251,13 @@
       _zkClient.delete(shard2NodePath);
     }
 
-    DeployedShard deployedShard = new DeployedShard(shardName, _searcher.getNumDoc(shardName));
+    int shardSize;
+    try {
+      shardSize = _server.shardSize(shardName);
+    } catch (Throwable t) {
+      throw new KattaException("Error measuring shard size for " + shardName, t);
+    }
+    DeployedShard deployedShard = new DeployedShard(shardName, shardSize);
     _zkClient.createEphemeral(shard2NodePath, deployedShard);
   }
 
@@ -328,14 +316,20 @@
           _zkClient.deleteIfExists(shard2NodePath);
           _zkClient.deleteIfExists(shard2ErrorPath);
         }
-      } catch (Exception e) {
-        LOG.warn("could'nt cleanup zk ephemeral pathes: " + e.getMessage());
+      } catch (Throwable t) {
+        LOG.warn("could'nt cleanup zk ephemeral pathes: " + t.getMessage());
       }
       _timer.cancel();
       _zkClient.unsubscribeAll();
       _zkClient.close();
       _rpcServer.stop();
       _rpcServer = null;
+      try {
+        _server.shutdown();
+      } catch (Throwable t) {
+        LOG.error("Error shutting down server", t);
+      }
+      _server = null;
     } finally {
       _zkClient.getEventLock().unlock();
     }
@@ -376,7 +370,7 @@
     int tryCount = 10000;
     while (_rpcServer == null) {
       try {
-        _rpcServer = RPC.getServer(this, "0.0.0.0", serverPort, new Configuration());
+        _rpcServer = RPC.getServer(_server, "0.0.0.0", serverPort, new Configuration());
         LOG.info("search server started on : " + hostName + ":" + serverPort);
         _searchServerPort = serverPort;
       } catch (final BindException e) {
@@ -390,7 +384,6 @@
         throw new RuntimeException("unable to create rpc search server", e);
       }
     }
-    _searcher = new KattaMultiSearcher(_nodeName);
     try {
       _rpcServer.start();
     } catch (final IOException e) {
@@ -403,124 +396,9 @@
     return new File(_shardsFolder, shardName);
   }
 
-//  /*
-//   * Reads AssignedShard data from ZooKeeper
-//   */
-//  private AssignedShard readAssignedShard(final String shardName) throws KattaException {
-//    final AssignedShard assignedShard = new AssignedShard();
-//    _zkClient.readData(ZkPathes.getNode2ShardPath(_nodeName, shardName), assignedShard);
-//    return assignedShard;
-//  }
 
-  public HitsMapWritable search(final QueryWritable query, final DocumentFrequenceWritable freqs, final String[] shards)
-      throws IOException {
-    return search(query, freqs, shards, Integer.MAX_VALUE - 1);
-  }
 
-  public HitsMapWritable search(final QueryWritable query, final DocumentFrequenceWritable freqs, final String[] shards,
-      final int count) throws IOException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("You are searching with the query: '" + query.getQuery() + "'");
-    }
 
-    Query luceneQuery = query.getQuery();
-   
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Lucene query: " + luceneQuery.toString());
-    }
-
-    long completeSearchTime = 0;
-    final HitsMapWritable result = new net.sf.katta.node.HitsMapWritable(_nodeName);
-    if (_searcher != null) {
-      long start = 0;
-      if (LOG.isDebugEnabled()) {
-        start = System.currentTimeMillis();
-      }
-      _searcher.search(luceneQuery, freqs, shards, result, count);
-      if (LOG.isDebugEnabled()) {
-        final long end = System.currentTimeMillis();
-        LOG.debug("Search took " + (end - start) / 1000.0 + "sec.");
-        completeSearchTime += (end - start);
-      }
-    } else {
-      LOG.error("No searcher for index found on '" + _nodeName + "'.");
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Complete search took " + completeSearchTime / 1000.0 + "sec.");
-      final DataOutputBuffer buffer = new DataOutputBuffer();
-      result.write(buffer);
-      LOG.debug("Result size to transfer: " + buffer.getLength());
-    }
-    return result;
-  }
-
-  public long getProtocolVersion(final String protocol, final long clientVersion) throws IOException {
-    return _protocolVersion;
-  }
-
-  public DocumentFrequenceWritable getDocFreqs(final QueryWritable input, final String[] shards) throws IOException {
-    Query luceneQuery = input.getQuery();
-
-    final Query rewrittenQuery = _searcher.rewrite(luceneQuery, shards);
-    final DocumentFrequenceWritable docFreqs = new DocumentFrequenceWritable();
-
-    final HashSet<Term> termSet = new HashSet<Term>();
-    rewrittenQuery.extractTerms(termSet);
-    int numDocs = 0;
-    for (final String shard : shards) {
-    final java.util.Iterator<Term> termIterator = termSet.iterator();
-      while (termIterator.hasNext()) {
-        final Term term = termIterator.next();
-        final int docFreq = _searcher.docFreq(shard, term);
-        docFreqs.put(term.field(), term.text(), docFreq);
-      }
-      numDocs += _searcher.getNumDoc(shard);
-    }
-    docFreqs.setNumDocs(numDocs);
-    return docFreqs;
-  }
-
-  @SuppressWarnings("unchecked")
-  public MapWritable getDetails(final String shard, final int docId) throws IOException {
-    final MapWritable result = new MapWritable();
-    final Document doc = _searcher.doc(shard, docId);
-    final List<Fieldable> fields = doc.getFields();
-    for (final Fieldable field : fields) {
-      final String name = field.name();
-      if (field.isBinary()) {
-        final byte[] binaryValue = field.binaryValue();
-        result.put(new Text(name), new BytesWritable(binaryValue));
-      } else {
-        final String stringValue = field.stringValue();
-        result.put(new Text(name), new Text(stringValue));
-      }
-    }
-    return result;
-  }
-
-  public MapWritable getDetails(final String shard, final int docId, final String[] fieldNames) throws IOException {
-    final MapWritable result = new MapWritable();
-    final Document doc = _searcher.doc(shard, docId);
-    for (final String fieldName : fieldNames) {
-      final Field field = doc.getField(fieldName);
-      if (field != null) {
-        if (field.isBinary()) {
-          final byte[] binaryValue = field.binaryValue();
-          result.put(new Text(fieldName), new BytesWritable(binaryValue));
-        } else {
-          final String stringValue = field.stringValue();
-          result.put(new Text(fieldName), new Text(stringValue));
-        }
-      }
-    }
-    return result;
-  }
-
-  public int getResultCount(final QueryWritable query, final String[] shards) throws IOException {
-    final DocumentFrequenceWritable docFreqs = getDocFreqs(query, shards);
-    return search(query, docFreqs, shards, 1).getTotalHits();
-  }
-
   @Override
   protected void finalize() throws Throwable {
     shutdown();
Index: src/main/java/net/sf/katta/node/AbstractServer.java
===================================================================
--- src/main/java/net/sf/katta/node/AbstractServer.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/main/java/net/sf/katta/node/AbstractServer.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,84 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.node;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.log4j.Logger;
+
+/**
+ * Maintains the current set of shards, as specified by the Node class.
+ */
+public abstract class AbstractServer implements INodeManaged {
+
+  private final static Logger LOG = Logger.getLogger(AbstractServer.class);
+
+  protected final Map<String, File> _shards = new ConcurrentHashMap<String, File>();
+  protected String _nodeName;
+
+  public long getProtocolVersion(final String protocol, final long clientVersion) throws IOException {
+    return 0L;
+  }
+
+  public void setNodeName(String nodeName) {
+    _nodeName = nodeName;
+  }
+
+  
+  /**
+   * Add a shard.
+   * 
+   * @param shardName
+   * @param indexSearcher
+   * @throws IOException
+   */
+  public void addShard(final String shardName, final File shardDir) throws IOException {
+    LOG.debug(_nodeName + " got shard " + shardName);
+    if (!shardDir.exists()) {
+      throw new IOException("Shard " + shardName + " dir " + shardDir.getAbsolutePath() + " does not exist!");
+    }
+    if (!shardDir.canRead()) {
+      throw new IOException("Can not read shard " + shardName + " dir " + shardDir.getAbsolutePath() + "!");
+    }
+    _shards.put(shardName, shardDir);
+  }
+
+  /**
+   * Remove a shard.
+   */
+  public void removeShard(final String shardName) throws IOException {
+    LOG.debug(_nodeName + " removing shard " + shardName);
+    _shards.remove(shardName);
+  }
+  
+  /**
+   * Returns the size of a shard. The units are server-implementation specific.
+   * 
+   * @param shardName The shard to measure.
+   * @return the size of the shard.
+   */
+  public abstract int shardSize(final String shardName) throws Exception;
+  
+
+  /**
+   * Release all resources. No further calls will be made after this one.
+   */
+  public abstract void shutdown() throws IOException;
+  
+}
Index: src/main/java/net/sf/katta/index/DeployedShard.java
===================================================================
--- src/main/java/net/sf/katta/index/DeployedShard.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/main/java/net/sf/katta/index/DeployedShard.java	(.../trunk/katta)	(revision 10997)
@@ -24,7 +24,7 @@
 public class DeployedShard implements Writable {
 
   private String _shardName;
-  private int _numOfDocs;
+  private int _shardSize;
   private long _deployTime = System.currentTimeMillis();
 
   public DeployedShard() {
@@ -33,19 +33,19 @@
 
   public DeployedShard(final String shardName, final int numOfDocs) {
     _shardName = shardName;
-    _numOfDocs = numOfDocs;
+    _shardSize = numOfDocs;
   }
 
   public void readFields(final DataInput in) throws IOException {
     _shardName = in.readUTF();
     _deployTime = in.readLong();
-    _numOfDocs = in.readInt();
+    _shardSize = in.readInt();
   }
 
   public void write(final DataOutput out) throws IOException {
     out.writeUTF(_shardName);
     out.writeLong(_deployTime);
-    out.writeInt(_numOfDocs);
+    out.writeInt(_shardSize);
   }
 
   public String getShardName() {
@@ -57,7 +57,7 @@
   }
 
   public int getNumOfDocs() {
-    return _numOfDocs;
+    return _shardSize;
   }
 
 }
Index: src/main/java/net/sf/katta/index/IndexMetaData.java
===================================================================
--- src/main/java/net/sf/katta/index/IndexMetaData.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/main/java/net/sf/katta/index/IndexMetaData.java	(.../trunk/katta)	(revision 10997)
@@ -25,7 +25,6 @@
 public class IndexMetaData implements Writable {
 
   private Text _path = new Text();
-  private Text _analyzerClassName = new Text();
   private int _replicationLevel;
 
   private IndexState _state;
@@ -35,9 +34,8 @@
     ANNOUNCED, DEPLOYED, ERROR, DEPLOYING, REPLICATING;
   }
 
-  public IndexMetaData(final String path, final String analyzerName, final int replicationLevel, final IndexState state) {
+  public IndexMetaData(final String path, final int replicationLevel, final IndexState state) {
     _path.set(path);
-    _analyzerClassName.set(analyzerName);
     _replicationLevel = replicationLevel;
     _state = state;
   }
@@ -48,7 +46,6 @@
 
   public void readFields(final DataInput in) throws IOException {
     _path.readFields(in);
-    _analyzerClassName.readFields(in);
     _replicationLevel = in.readInt();
     _state = IndexState.values()[in.readByte()];
     if (_state == IndexState.ERROR) {
@@ -58,7 +55,6 @@
 
   public void write(final DataOutput out) throws IOException {
     _path.write(out);
-    _analyzerClassName.write(out);
     out.writeInt(_replicationLevel);
     out.writeByte(_state.ordinal());
     if (_state == IndexState.ERROR) {
@@ -70,10 +66,6 @@
     return _path.toString();
   }
 
-  public String getAnalyzerClassName() {
-    return _analyzerClassName.toString();
-  }
-
   public IndexState getState() {
     return _state;
   }
Index: src/main/java/net/sf/katta/index/indexer/merge/IndexMergeApplication.java
===================================================================
--- src/main/java/net/sf/katta/index/indexer/merge/IndexMergeApplication.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/main/java/net/sf/katta/index/indexer/merge/IndexMergeApplication.java	(.../trunk/katta)	(revision 10997)
@@ -126,7 +126,7 @@
       mergedIndex = mergedIndex.makeQualified(fileSystem);
       LOG.info("deploying new merged index: " + mergedIndex);
       IIndexDeployFuture deployFuture = deployClient.addIndex(mergedIndex.getName(), mergedIndex.toString()
-          + "/indexes", deployedIndexes.get(0).getAnalyzerClassName(), deployedIndexes.get(0).getReplicationLevel());
+          + "/indexes", deployedIndexes.get(0).getReplicationLevel());
       // TODO jz: just taking the analyzer and replication level from the
       // first is unclean
       // TODO jz: appending / indexes is suboptimal
Index: src/main/java/net/sf/katta/client/IClient.java
===================================================================
--- src/main/java/net/sf/katta/client/IClient.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/main/java/net/sf/katta/client/IClient.java	(.../trunk/katta)	(revision 10997)
@@ -1,170 +0,0 @@
-/**
- * Copyright 2008 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.sf.katta.client;
-
-import java.util.List;
-
-import net.sf.katta.node.Hit;
-import net.sf.katta.node.Hits;
-import net.sf.katta.node.IQuery;
-import net.sf.katta.util.KattaException;
-
-import org.apache.hadoop.io.MapWritable;
-import org.apache.lucene.search.Query;
-
-/**
- * Client for searching document indices deployed on a katta cluster.
- * <p>
- * 
- * You provide a {@link IQuery} and the name of the deployed indices, and get
- * back {@link Hits} which contains multiple {@link Hit} objects as the results.
- * <br>
- * See {@link #search(IQuery, String[], int)}.
- * <p>
- * 
- * The details of a hit-document can be retrieved through the
- * {@link #getDetails(Hit, String[])} method.
- * 
- * @see Hit
- * @see Hits
- * @see IQuery
- */
-public interface IClient {
-
-  /**
-   * Searches with a given query in the supplied indexes for an almost unlimited
-   * ({@link Integer.MAX_VALUE}) amount of results.
-   * 
-   * If this method might has poor performance try to limit results with
-   * {@link #search(IQuery, String[], int)}.
-   * 
-   * @param query
-   *          The query to search with.
-   * @param indexNames
-   *          A list of index names to search in.
-   * @return A object that capsulates all results.
-   * @throws KattaException
-   */
-  public abstract Hits search(Query query, String[] indexNames) throws KattaException;
-
-  @Deprecated
-  public abstract Hits search(IQuery query, String[] indexNames) throws KattaException;
-  
-  
-
-  /**
-   * Searches with a given query in the supplied indexes for a limited amount of
-   * results.
-   * 
-   * @param query
-   *          The query to search with.
-   * @param indexNames
-   *          A list of index names to search in.
-   * @param count
-   *          The count of results that should be returned.
-   * @return A object that capsulates all results.
-   * @throws KattaException
-   */
-  public abstract Hits search(Query query, String[] indexNames, int count) throws KattaException;
-  
-  @Deprecated
-  public abstract Hits search(IQuery query, String[] indexNames, int count) throws KattaException;
-
-  /**
-   * Gets all the details to a hit.
-   * 
-   * @param hit
-   *          The {@link Hit} from that all fields should be returned.
-   * @return All fields to a {@link Hit} as field name and field value pairs.
-   * @throws KattaException
-   *           If indexes can't be searched.
-   */
-  public abstract MapWritable getDetails(Hit hit) throws KattaException;
-
-  /**
-   * Gets a specific details to a hit.
-   * 
-   * @param hit
-   *          The {@link Hit} from that all fields should be returned.
-   * @param fields
-   *          The names of the fields from that the value should be returned.
-   * @return The supplied field to a {@link Hit} as field name and field value
-   *         pair.
-   * @throws KattaException
-   *           If indexes can't be searched.
-   */
-  public abstract MapWritable getDetails(Hit hit, String[] fields) throws KattaException;
-
-  /**
-   * Gets list of all details for the given list of hits. The details are retrieved in
-   * parallel rather than getting them one by one. Thus using this method is the preferred
-   * way of getting the details of multiple hits.
-   * 
-   * @param hits
-   *          The list of hits from that all fields should be returned.
-   * @return The list of details for given hits.
-   * @throws KattaException
-   *           If indexes can't be searched.
-   * @throws InterruptedException
-   *           If the current thread got interrupted.
-   */
-  public List<MapWritable> getDetails(List<Hit> hits) throws KattaException, InterruptedException;
-
-  /**
-   * Gets list of details for the given list of hits. The details are retrieved in
-   * parallel rather than getting them one by one. Thus using this method is the preferred
-   * way of getting the details of multiple hits.
-   * 
-   * @param hits
-   *          The list of hits from that all fields should be returned.
-   * @param fields
-   *          The field names of which the value should be returned.
-   * @return The list of details for given hits.
-   * @throws KattaException
-   *           If indexes can't be searched.
-   * @throws InterruptedException
-   *           If the current thread got interrupted.
-   */
-  public List<MapWritable> getDetails(List<Hit> hits, final String[] fields) throws KattaException, InterruptedException;
-
-  /**
-   * The overall queries per minute.
-   * 
-   * @return A number that represents the queries per minute in the last minute.
-   */
-  public abstract float getQueryPerMinute();
-
-  /**
-   * Gets only the result count to a query.
-   * 
-   * @param query
-   *          The query to search with.
-   * @param indexNames
-   *          A list of index names to search in.
-   * @return A number that represents the overall result count to a query.
-   * @throws KattaException
-   */
-  public abstract int count(Query query, String[] indexNames) throws KattaException;
-  
-  @Deprecated
-  public abstract int count(IQuery query, String[] indexNames) throws KattaException;
-
-  /**
-   * Closes down the client.
-   */
-  public abstract void close();
-
-}
\ No newline at end of file
Index: src/main/java/net/sf/katta/client/ILuceneClient.java
===================================================================
--- src/main/java/net/sf/katta/client/ILuceneClient.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/main/java/net/sf/katta/client/ILuceneClient.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,170 @@
+/**
+ * Copyright 2008 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.client;
+
+import java.util.List;
+
+import net.sf.katta.node.Hit;
+import net.sf.katta.node.Hits;
+import net.sf.katta.node.IQuery;
+import net.sf.katta.util.KattaException;
+
+import org.apache.hadoop.io.MapWritable;
+import org.apache.lucene.search.Query;
+
+/**
+ * Client for searching document indices deployed on a katta cluster.
+ * <p>
+ * 
+ * You provide a {@link IQuery} and the name of the deployed indices, and get
+ * back {@link Hits} which contains multiple {@link Hit} objects as the results.
+ * <br>
+ * See {@link #search(IQuery, String[], int)}.
+ * <p>
+ * 
+ * The details of a hit-document can be retrieved through the
+ * {@link #getDetails(Hit, String[])} method.
+ * 
+ * @see Hit
+ * @see Hits
+ * @see IQuery
+ */
+public interface ILuceneClient {
+
+  /**
+   * Searches with a given query in the supplied indexes for an almost unlimited
+   * ({@link Integer.MAX_VALUE}) amount of results.
+   * 
+   * If this method might has poor performance try to limit results with
+   * {@link #search(IQuery, String[], int)}.
+   * 
+   * @param query
+   *          The query to search with.
+   * @param indexNames
+   *          A list of index names to search in.
+   * @return A object that capsulates all results.
+   * @throws KattaException
+   */
+  public Hits search(Query query, String[] indexNames) throws KattaException;
+
+  @Deprecated
+  public Hits search(IQuery query, String[] indexNames) throws KattaException;
+  
+  
+
+  /**
+   * Searches with a given query in the supplied indexes for a limited amount of
+   * results.
+   * 
+   * @param query
+   *          The query to search with.
+   * @param indexNames
+   *          A list of index names to search in.
+   * @param count
+   *          The count of results that should be returned.
+   * @return A object that capsulates all results.
+   * @throws KattaException
+   */
+  public Hits search(Query query, String[] indexNames, int count) throws KattaException;
+  
+  @Deprecated
+  public Hits search(IQuery query, String[] indexNames, int count) throws KattaException;
+
+  /**
+   * Gets all the details to a hit.
+   * 
+   * @param hit
+   *          The {@link Hit} from that all fields should be returned.
+   * @return All fields to a {@link Hit} as field name and field value pairs.
+   * @throws KattaException
+   *           If indexes can't be searched.
+   */
+  public MapWritable getDetails(Hit hit) throws KattaException;
+
+  /**
+   * Gets a specific details to a hit.
+   * 
+   * @param hit
+   *          The {@link Hit} from that all fields should be returned.
+   * @param fields
+   *          The names of the fields from that the value should be returned.
+   * @return The supplied field to a {@link Hit} as field name and field value
+   *         pair.
+   * @throws KattaException
+   *           If indexes can't be searched.
+   */
+  public MapWritable getDetails(Hit hit, String[] fields) throws KattaException;
+
+  /**
+   * Gets list of all details for the given list of hits. The details are retrieved in
+   * parallel rather than getting them one by one. Thus using this method is the preferred
+   * way of getting the details of multiple hits.
+   * 
+   * @param hits
+   *          The list of hits from that all fields should be returned.
+   * @return The list of details for given hits.
+   * @throws KattaException
+   *           If indexes can't be searched.
+   * @throws InterruptedException
+   *           If the current thread got interrupted.
+   */
+  public List<MapWritable> getDetails(List<Hit> hits) throws KattaException, InterruptedException;
+
+  /**
+   * Gets list of details for the given list of hits. The details are retrieved in
+   * parallel rather than getting them one by one. Thus using this method is the preferred
+   * way of getting the details of multiple hits.
+   * 
+   * @param hits
+   *          The list of hits from that all fields should be returned.
+   * @param fields
+   *          The field names of which the value should be returned.
+   * @return The list of details for given hits.
+   * @throws KattaException
+   *           If indexes can't be searched.
+   * @throws InterruptedException
+   *           If the current thread got interrupted.
+   */
+  public List<MapWritable> getDetails(List<Hit> hits, final String[] fields) throws KattaException, InterruptedException;
+
+  /**
+   * The overall queries per minute.
+   * 
+   * @return A number that represents the queries per minute in the last minute.
+   */
+  public double getQueryPerMinute();
+
+  /**
+   * Gets only the result count to a query.
+   * 
+   * @param query
+   *          The query to search with.
+   * @param indexNames
+   *          A list of index names to search in.
+   * @return A number that represents the overall result count to a query.
+   * @throws KattaException
+   */
+  public int count(Query query, String[] indexNames) throws KattaException;
+  
+  @Deprecated
+  public int count(IQuery query, String[] indexNames) throws KattaException;
+
+  /**
+   * Closes down the client.
+   */
+  public void close();
+
+}
\ No newline at end of file
Index: src/main/java/net/sf/katta/client/IDeployClient.java
===================================================================
--- src/main/java/net/sf/katta/client/IDeployClient.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/main/java/net/sf/katta/client/IDeployClient.java	(.../trunk/katta)	(revision 10997)
@@ -23,7 +23,7 @@
 
 public interface IDeployClient {
 
-  IIndexDeployFuture addIndex(final String name, final String path, final String analyzerClass,
+  IIndexDeployFuture addIndex(final String name, final String path,
       final int replicationLevel) throws KattaException;
 
   void removeIndex(final String name) throws KattaException;
Index: src/main/java/net/sf/katta/client/IMapFileClient.java
===================================================================
--- src/main/java/net/sf/katta/client/IMapFileClient.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/main/java/net/sf/katta/client/IMapFileClient.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,43 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.client;
+
+import java.io.IOException;
+import java.util.List;
+
+import net.sf.katta.util.KattaException;
+
+/**
+ * The public interface to the front end of the MapFile server.
+ */
+public interface IMapFileClient {
+
+  /**
+   * Get all entries with the given key.
+   * 
+   * @param key The entry(s) to look up.
+   * @param indexNames The MapFiles to search.
+   * @return All the entries with the given key.
+   * @throws IOException
+   */
+  public List<String> get(String key, final String[] indexNames) throws KattaException;
+
+  /**
+   * Closes down the client.
+   */
+  public void close();
+
+}
\ No newline at end of file
Index: src/main/java/net/sf/katta/client/LuceneClient.java
===================================================================
--- src/main/java/net/sf/katta/client/LuceneClient.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/main/java/net/sf/katta/client/LuceneClient.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,292 @@
+/**
+ * Copyright 2008 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.client;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import net.sf.katta.node.DocumentFrequenceWritable;
+import net.sf.katta.node.Hit;
+import net.sf.katta.node.Hits;
+import net.sf.katta.node.HitsMapWritable;
+import net.sf.katta.node.ILuceneServer;
+import net.sf.katta.node.IQuery;
+import net.sf.katta.node.QueryWritable;
+import net.sf.katta.util.KattaException;
+import net.sf.katta.util.ZkConfiguration;
+
+import org.apache.hadoop.io.MapWritable;
+import org.apache.log4j.Logger;
+import org.apache.lucene.analysis.KeywordAnalyzer;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.Query;
+
+/**
+ * Default implementation of {@link ILuceneClient}.
+ */
+public class LuceneClient implements ILuceneClient {
+
+  protected final static Logger LOG = Logger.getLogger(LuceneClient.class);
+
+  @SuppressWarnings("unused")
+  private static Method getMethod(String name, Class<?>... parameterTypes) {
+    try {
+      return ILuceneServer.class.getMethod("search", parameterTypes);
+    } catch (NoSuchMethodException e) {
+      throw new RuntimeException("Could not find method " + name + "(" + Arrays.asList(parameterTypes) + ") in ILuceneSearch!");
+    }
+  }
+  
+  private Client kattaClient;
+//  private static MethodCache methodCache = new MethodCache(ILuceneSearch.class);
+  
+  public LuceneClient(final INodeSelectionPolicy nodeSelectionPolicy) throws KattaException {
+    kattaClient = new Client(ILuceneServer.class, nodeSelectionPolicy);
+  }
+
+  public LuceneClient() throws KattaException {
+    kattaClient = new Client(ILuceneServer.class);
+  }
+
+  public LuceneClient(final INodeSelectionPolicy policy, final ZkConfiguration config) throws KattaException {
+    kattaClient = new Client(ILuceneServer.class, policy, config);
+  }
+
+
+  @Deprecated
+  /*
+   * @deprecated Old api uses IQuery what just transport a string, also uses
+   * only a KeywordAnalyzer.
+   */
+  public Hits search(final IQuery query, final String[] indexNames) throws KattaException {
+    return search(query, indexNames, Integer.MAX_VALUE);
+  }
+
+  public Hits search(final Query query, final String[] indexNames) throws KattaException {
+    return search(query, indexNames, Integer.MAX_VALUE);
+  }
+
+  @Deprecated
+  /*
+   * @deprecated Old api uses IQuery what just transport a string, also uses
+   * only a KeywordAnalyzer.
+   */
+  public Hits search(final IQuery query, final String[] indexNames, final int count) throws KattaException {
+    try {
+      final QueryParser luceneQueryParser = new QueryParser("field", new KeywordAnalyzer());
+      Query luceneQuery = luceneQueryParser.parse(query.getQuery());
+      return search(luceneQuery, indexNames, count);
+    } catch (ParseException e) {
+      throw new KattaException("Unable to parse Query: " + query.getQuery(), e);
+    }
+  }
+
+  private static final Method SEARCH_METHOD;
+  private static final int SEARCH_METHOD_SHARD_ARG_IDX = 2;
+  static {
+    try {
+      SEARCH_METHOD = ILuceneServer.class.getMethod("search", 
+              new Class[] { QueryWritable.class, DocumentFrequenceWritable.class, String[].class, Integer.TYPE });
+    } catch (NoSuchMethodException e) {
+      throw new RuntimeException("Could not find method search() in ILuceneSearch!");
+    }
+  }
+  
+  public Hits search(final Query query, final String[] indexNames, final int count) throws KattaException {
+    final DocumentFrequenceWritable docFreqs = getDocFrequencies(query, indexNames);
+    Collection<HitsMapWritable> allHits = kattaClient.broadcastToIndices(SEARCH_METHOD, SEARCH_METHOD_SHARD_ARG_IDX, indexNames, new QueryWritable(query), docFreqs, null, Integer.valueOf(count));
+    Hits result = new Hits();
+    for (HitsMapWritable hmw : allHits) {
+      Hits hits = hmw.getHits();
+      result.addTotalHits(hits.size());
+      result.addHits(hits.getHits());
+    }
+    long start = 0;
+    if (LOG.isDebugEnabled()) {
+      start = System.currentTimeMillis();
+    }
+    result.sort(count);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Time for sorting: " + (System.currentTimeMillis() - start) + " ms");
+    }
+    return result;
+  }
+  
+  
+  
+  
+  //  public int getResultCount(QueryWritable query, String[] shards) throws IOException;
+
+  private static final Method COUNT_METHOD;
+  private static final int COUNT_METHOD_SHARD_ARG_IDX = 1;
+  static {
+    try {
+      COUNT_METHOD = ILuceneServer.class.getMethod("getResultCount", 
+              new Class[] { QueryWritable.class, String[].class });
+    } catch (NoSuchMethodException e) {
+      throw new RuntimeException("Could not find method getResultCount() in ILuceneSearch!");
+    }
+  }
+
+  @Deprecated
+  public int count(final IQuery query, final String[] indexNames) throws KattaException {
+    try {
+      final QueryParser luceneQueryParser = new QueryParser("field", new KeywordAnalyzer());
+      Query luceneQuery = luceneQueryParser.parse(query.getQuery());
+      return count(luceneQuery, indexNames);
+    } catch (ParseException e) {
+      throw new KattaException("Unable to parse Query: " + query.getQuery(), e);
+    }
+  }
+
+  public int count(final Query query, final String[] indexNames) throws KattaException {
+    Collection<Integer> counts = kattaClient.broadcastToIndices(COUNT_METHOD, COUNT_METHOD_SHARD_ARG_IDX, indexNames, new QueryWritable(query), null);
+    int count = 0;
+    for (Integer n : counts) {
+      count += n.intValue();
+    }
+    return count;
+  }
+
+  
+  
+
+  private static final Method DOC_FREQ_METHOD;
+  private static final int DOC_FREQ_METHOD_SHARD_ARG_IDX = 1;
+  static {
+    try {
+      DOC_FREQ_METHOD = ILuceneServer.class.getMethod("getDocFreqs", 
+              new Class[] { QueryWritable.class, String[].class });
+    } catch (NoSuchMethodException e) {
+      throw new RuntimeException("Could not find method getDocFreqs() in ILuceneSearch!");
+    }
+  }
+
+  private DocumentFrequenceWritable getDocFrequencies(final Query query, final String[] indexNames)
+          throws KattaException {
+    Collection<DocumentFrequenceWritable> docFreqs = kattaClient.broadcastToIndices(DOC_FREQ_METHOD, DOC_FREQ_METHOD_SHARD_ARG_IDX, indexNames, new QueryWritable(query), null);
+    DocumentFrequenceWritable result = null;
+    for (DocumentFrequenceWritable df : docFreqs) {
+      if (result == null) {
+        // Start with first result.
+        result = df;
+      } else {
+        // Aggregate rest of results into first result.
+        result.addNumDocs(df.getNumDocs());
+        result.putAll(df.getAll());
+      }
+    }
+    if (result == null) {
+      result = new DocumentFrequenceWritable(); // TODO: ?
+    }
+    return result;
+  }
+
+  
+  
+  /*  public MapWritable getDetails(String[] shards, int docId, String[] fields) throws IOException;
+public MapWritable getDetails(String[] shards, int docId) throws IOException;
+*/
+  private static final Method GET_DETAILS_METHOD;
+  private static final Method GET_DETAILS_FIELDS_METHOD;
+  private static final int GET_DETAILS_METHOD_SHARD_ARG_IDX = 0;
+  private static final int GET_DETAILS_FIELDS_METHOD_SHARD_ARG_IDX = 0;
+  static {
+    try {
+      GET_DETAILS_METHOD = ILuceneServer.class.getMethod("getDetails", 
+              new Class[] { String[].class, Integer.TYPE });
+      GET_DETAILS_FIELDS_METHOD = ILuceneServer.class.getMethod("getDetails", 
+              new Class[] { String[].class, Integer.TYPE, String[].class });
+    } catch (NoSuchMethodException e) {
+      throw new RuntimeException("Could not find method getDetails() in ILuceneSearch!");
+    }
+  }
+  
+  public MapWritable getDetails(final Hit hit) throws KattaException {
+    return getDetails(hit, null);
+  }
+
+  public MapWritable getDetails(final Hit hit, final String[] fields) throws KattaException {
+    List<String> shards = new ArrayList<String>();
+    shards.add(hit.getShard());
+    int docId = hit.getDocId();
+    //
+    Object[] args;
+    Method method;
+    int shardArgIdx;
+    if (fields == null) {
+      args = new Object[] { null, Integer.valueOf(docId) };
+      method = GET_DETAILS_METHOD;
+      shardArgIdx = GET_DETAILS_METHOD_SHARD_ARG_IDX;
+    } else {
+      args = new Object[] { null, Integer.valueOf(docId), fields };
+      method = GET_DETAILS_FIELDS_METHOD;
+      shardArgIdx = GET_DETAILS_FIELDS_METHOD_SHARD_ARG_IDX;
+    }
+    Collection<MapWritable> detailsList = kattaClient.broadcastToShards(method, shardArgIdx, shards, args);
+    return detailsList.isEmpty() ? null : detailsList.iterator().next();
+  }
+
+  public List<MapWritable> getDetails(List<Hit> hits) throws KattaException, InterruptedException {
+    return getDetails(hits, null);
+  }
+
+  public List<MapWritable> getDetails(List<Hit> hits, final String[] fields) throws KattaException, InterruptedException {
+    ExecutorService executorService = Executors.newFixedThreadPool(Math.min(10, hits.size() + 1));
+    List<MapWritable> results = new ArrayList<MapWritable>();
+    List<Future<MapWritable>> futures = new ArrayList<Future<MapWritable>>();
+    for (final Hit hit : hits) {
+      futures.add(executorService.submit(new Callable<MapWritable>() {
+
+        @Override
+        public MapWritable call() throws Exception {
+          return getDetails(hit, fields);
+        }
+      }));
+    }
+
+    for (Future<MapWritable> future : futures) {
+      try {
+        results.add(future.get());
+      } catch (ExecutionException e) {
+        throw new KattaException("Could not get hit details.", e.getCause());
+      }
+    }
+
+    executorService.shutdown();
+    
+    return results;
+  }
+
+  
+  public double getQueryPerMinute() {
+    return kattaClient.getQueryPerMinute();
+  }
+
+  public void close() {
+    kattaClient.close();
+  }
+
+}
Index: src/main/java/net/sf/katta/client/DeployClient.java
===================================================================
--- src/main/java/net/sf/katta/client/DeployClient.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/main/java/net/sf/katta/client/DeployClient.java	(.../trunk/katta)	(revision 10997)
@@ -40,12 +40,12 @@
     }
   }
 
-  public IIndexDeployFuture addIndex(String name, String path, String analyzerClass, int replicationLevel)
+  public IIndexDeployFuture addIndex(String name, String path, int replicationLevel)
       throws KattaException {
     final String indexPath = ZkPathes.getIndexPath(name);
     validateIndexName(name, indexPath);
 
-    final IndexMetaData indexMetaData = new IndexMetaData(path, analyzerClass, replicationLevel,
+    final IndexMetaData indexMetaData = new IndexMetaData(path, replicationLevel,
         IndexMetaData.IndexState.ANNOUNCED);
     _zkClient.create(indexPath, indexMetaData);
     return new IndexDeployFuture(_zkClient, name, indexMetaData);
Index: src/main/java/net/sf/katta/client/MapFileClient.java
===================================================================
--- src/main/java/net/sf/katta/client/MapFileClient.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ src/main/java/net/sf/katta/client/MapFileClient.java	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,96 @@
+/**
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.katta.client;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import net.sf.katta.node.IMapFileServer;
+import net.sf.katta.node.TextArrayWritable;
+import net.sf.katta.util.KattaException;
+import net.sf.katta.util.ZkConfiguration;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.log4j.Logger;
+
+/**
+ * The front end to the MapFile server.
+ */
+public class MapFileClient implements IMapFileClient {
+
+  @SuppressWarnings("unused")
+  private final static Logger LOG = Logger.getLogger(MapFileClient.class);
+
+  @SuppressWarnings("unused")
+  private static Method getMethod(String name, Class<?>... parameterTypes) {
+    try {
+      return IMapFileServer.class.getMethod("search", parameterTypes);
+    } catch (NoSuchMethodException e) {
+      throw new RuntimeException("Could not find method " + name + "(" + Arrays.asList(parameterTypes) + ") in IMapFileServer!");
+    }
+  }
+  
+  private Client kattaClient;
+//  private static MethodCache methodCache = new MethodCache(ILuceneSearch.class);
+  
+  public MapFileClient(final INodeSelectionPolicy nodeSelectionPolicy) throws KattaException {
+    kattaClient = new Client(IMapFileServer.class, nodeSelectionPolicy);
+  }
+
+  public MapFileClient() throws KattaException {
+    kattaClient = new Client(IMapFileServer.class);
+  }
+
+  public MapFileClient(final INodeSelectionPolicy policy, final ZkConfiguration config) throws KattaException {
+    kattaClient = new Client(IMapFileServer.class, policy, config);
+  }
+
+
+//  public List<Writable> get(WritableComparable<?> key, String[] shards) throws IOException {
+
+  private static final Method GET_METHOD;
+  private static final int GET_METHOD_SHARD_ARG_IDX = 1;
+  static {
+    try {
+      GET_METHOD = IMapFileServer.class.getMethod("get", 
+              new Class[] { Text.class, String[].class });
+    } catch (NoSuchMethodException e) {
+      throw new RuntimeException("Could not find method get() in IMapFileServer!");
+    }
+  }
+  
+  public List<String> get(final String key, final String[] indexNames) throws KattaException {
+    Collection<TextArrayWritable> entries = kattaClient.broadcastToIndices(GET_METHOD, GET_METHOD_SHARD_ARG_IDX, indexNames, new Text(key), null);
+    List<String> results = new ArrayList<String>();
+    for (TextArrayWritable taw : entries) {
+      for (Writable w : taw.array.get()) {
+        Text text = (Text) w;
+        results.add(text.toString());
+      }
+    }
+    return results;
+  }
+  
+
+  public void close() {
+    kattaClient.close();
+  }
+
+}
Index: src/main/java/net/sf/katta/client/Client.java
===================================================================
--- src/main/java/net/sf/katta/client/Client.java	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 10997)
+++ src/main/java/net/sf/katta/client/Client.java	(.../trunk/katta)	(revision 10997)
@@ -16,6 +16,7 @@
 package net.sf.katta.client;
 
 import java.io.IOException;
+import java.lang.reflect.Method;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -29,15 +30,10 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import net.sf.katta.index.IndexMetaData;
-import net.sf.katta.node.DocumentFrequenceWritable;
-import net.sf.katta.node.Hit;
-import net.sf.katta.node.Hits;
-import net.sf.katta.node.HitsMapWritable;
-import net.sf.katta.node.IQuery;
-import net.sf.katta.node.ISearch;
-import net.sf.katta.node.QueryWritable;
 import net.sf.katta.util.CollectionUtil;
 import net.sf.katta.util.KattaException;
 import net.sf.katta.util.ZkConfiguration;
@@ -47,31 +43,27 @@
 import net.sf.katta.zk.ZkPathes;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.VersionedProtocol;
 import org.apache.log4j.Logger;
-import org.apache.lucene.analysis.KeywordAnalyzer;
-import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.Query;
 
 /**
- * Default implementation of {@link IClient}.
  * 
  */
-public class Client implements IClient {
+public class Client {
 
   protected final static Logger LOG = Logger.getLogger(Client.class);
 
   protected final ZKClient _zkClient;
-
+  protected final Class<? extends VersionedProtocol> _serverClass;
+  
   private final IndexStateListener _indexStateListener = new IndexStateListener();
   private final IndexPathListener _indexPathChangeListener = new IndexPathListener();
   private final ShardNodeListener _shardNodeListener = new ShardNodeListener();
 
   protected final Map<String, List<String>> _indexToShards = new HashMap<String, List<String>>();
   // TODO: jz remove node proxies if not needed anymore
-  protected final Map<String, ISearch> _node2SearchProxyMap = new HashMap<String, ISearch>();
+  protected final Map<String, VersionedProtocol> _node2ProxyMap = new HashMap<String, VersionedProtocol>();
 
   protected final INodeSelectionPolicy _selectionPolicy;
   private long _queryCount = 0;
@@ -79,19 +71,20 @@
 
   private Configuration _hadoopConf = new Configuration();
 
-  public Client(final INodeSelectionPolicy nodeSelectionPolicy) throws KattaException {
-    this(nodeSelectionPolicy, new ZkConfiguration());
+  public Client(Class<? extends VersionedProtocol> serverClass, final INodeSelectionPolicy nodeSelectionPolicy) throws KattaException {
+    this(serverClass, nodeSelectionPolicy, new ZkConfiguration());
   }
 
-  public Client() throws KattaException {
-    this(new DefaultNodeSelectionPolicy(), new ZkConfiguration());
+  public Client(Class<? extends VersionedProtocol> serverClass) throws KattaException {
+    this(serverClass, new DefaultNodeSelectionPolicy(), new ZkConfiguration());
   }
 
-  public Client(final INodeSelectionPolicy policy, final ZkConfiguration config) throws KattaException {
+  public Client(Class<? extends VersionedProtocol> serverClass, final INodeSelectionPolicy policy, final ZkConfiguration config) throws KattaException {
     _hadoopConf.set("ipc.client.timeout", "2500");
     _hadoopConf.set("ipc.client.connect.max.retries", "2");
     // TODO jz: make configurable
 
+    _serverClass = serverClass;
     _selectionPolicy = policy;
     _zkClient = new ZKClient(config);
     try {
@@ -106,6 +99,8 @@
     _start = System.currentTimeMillis();
   }
 
+  // --------------- Proxy handling ----------------------
+  
   protected void updateSelectionPolicy(final String shardName, List<String> nodes) {
     List<String> connectedNodes = eastablishNodeProxiesIfNecessary(nodes);
     _selectionPolicy.update(shardName, connectedNodes);
@@ -114,9 +109,9 @@
   private List<String> eastablishNodeProxiesIfNecessary(List<String> nodes) {
     List<String> connectedNodes = new ArrayList<String>(nodes);
     for (String node : nodes) {
-      if (!_node2SearchProxyMap.containsKey(node)) {
+      if (!_node2ProxyMap.containsKey(node)) {
         try {
-          _node2SearchProxyMap.put(node, createNodeProxy(node));
+          _node2ProxyMap.put(node, createNodeProxy(node));
         } catch (Exception e) {
           connectedNodes.remove(node);
           LOG.warn("could not create proxy for node '" + node + "' - " + e.getClass().getSimpleName());
@@ -126,7 +121,7 @@
     return connectedNodes;
   }
 
-  protected ISearch createNodeProxy(final String node) throws IOException {
+  protected VersionedProtocol createNodeProxy(final String node) throws IOException {
     LOG.debug("creating proxy for node: " + node);
 
     String[] hostName_port = node.split(":");
@@ -137,7 +132,7 @@
     final String hostName = hostName_port[0];
     final String port = hostName_port[1];
     final InetSocketAddress inetSocketAddress = new InetSocketAddress(hostName, Integer.parseInt(port));
-    return (ISearch) RPC.getProxy(ISearch.class, 0L, inetSocketAddress, _hadoopConf);
+    return RPC.getProxy(_serverClass, 0L, inetSocketAddress, _hadoopConf);
   }
 
   protected void removeIndexes(List<String> indexes) {
@@ -180,90 +175,160 @@
             || indexMetaData.getState() == IndexMetaData.IndexState.REPLICATING;
   }
 
-  @Deprecated
-  /*
-   * @deprecated Old api uses IQuery what just transport a string, also uses
-   * only a KeywordAnalyzer.
-   */
-  public Hits search(final IQuery query, final String[] indexNames) throws KattaException {
-    return search(query, indexNames, Integer.MAX_VALUE);
-  }
+  
+  // --------------- Distributed calls to servers ----------------------
 
-  public Hits search(final Query query, final String[] indexNames) throws KattaException {
-    return search(query, indexNames, Integer.MAX_VALUE);
-  }
 
-  @Deprecated
-  /*
-   * @deprecated Old api uses IQuery what just transport a string, also uses
-   * only a KeywordAnalyzer.
+
+  /**
+   * Broadcast a method call to all indices. Return all the results in
+   * a Collection.
+   * 
+   * @param method The server's method to call.
+   * @param shardArrayParamIndex Which parameter of the method call, if any,
+   *     that should be replaced with the shards to search. This is
+   *     an array of Strings, with a different value for each node / server.
+   *     Pass in -1 to disable.
+   * @param args The arguments to pass to the method when run on the server.
+   * @return 
+   * @throws KattaException
    */
-  public Hits search(final IQuery query, final String[] indexNames, final int count) throws KattaException {
-    try {
-      final QueryParser luceneQueryParser = new QueryParser("field", new KeywordAnalyzer());
-      Query luceneQuery = luceneQueryParser.parse(query.getQuery());
-      return search(luceneQuery, indexNames, count);
-    } catch (ParseException e) {
-      throw new KattaException("Unable to parse Query: " + query.getQuery(), e);
-    }
+  public <T> Collection<T> broadcast(Method method, int shardArrayParamIndex, Object... args) throws KattaException {
+    return broadcastToShards(method, shardArrayParamIndex, null, args);
   }
+  
+  /**
+   * @param method
+   * @param shardArrayIndex
+   * @param indices
+   * @param args
+   * @return
+   * @throws KattaException
+   */
+  public <T> Collection<T> broadcastToIndices(Method method, int shardArrayIndex, String[] indices, Object... args) throws KattaException {
+    Map<String, List<String>> nodeShardsMap = getNode2ShardsMap(indices);
+    return broadcast(method, shardArrayIndex, nodeShardsMap, args);
+  }
+    
+  public <T> T singlecast(Method method, int shardArrayParamIndex, String shard, Object... args) throws KattaException {
+    List<String> shards = new ArrayList<String>();
+    shards.add(shard);
+    Collection<T> results = broadcastToShards(method, shardArrayParamIndex, shards, args);
+    return results.isEmpty() ? null : results.iterator().next();
+  }
 
-  public Hits search(final Query query, final String[] indexNames, final int count) throws KattaException {
-    final Map<String, List<String>> nodeShardsMap = getNode2ShardsMap(indexNames);
-    final Hits result = new Hits();
-    final DocumentFrequenceWritable docFreqs = getDocFrequencies(query, nodeShardsMap);
-
-    List<NodeInteraction> nodeInteractions = new ArrayList<NodeInteraction>();
-    for (final String node : nodeShardsMap.keySet()) {
-      nodeInteractions.add(new SearchInteraction(node, nodeShardsMap, query, docFreqs, result, count));
+  public <T> Collection<T> broadcastToShards(Method method, int shardArrayParamIndex, List<String> shards, Object... args) throws KattaException {
+    if (shards == null) {
+      // If no shards specified, search all shards.
+      shards = new ArrayList<String>(_indexToShards.keySet());
+    } 
+    final Map<String, List<String>> nodeShardsMap = _selectionPolicy.createNode2ShardsMap(shards);
+    return broadcast(method, shardArrayParamIndex, nodeShardsMap, args);
+ }
+  
+  private <T> Collection<T> broadcast(Method method, int shardArrayParamIndex, Map<String, List<String>> nodeShardsMap, Object... args) throws KattaException {
+    _queryCount++;
+    Collection<T> results = new ArrayList<T>();
+    /*
+     * Validate inputs.
+     */
+    if (method == null || args == null) {
+      throw new IllegalArgumentException("Null method or args!");
     }
-    execute(nodeInteractions);
-
+    Class<?>[] types = method.getParameterTypes();
+    if (args.length != types.length) {
+      throw new IllegalArgumentException("Wrong number of args: found " + args.length + ", expected " + types.length + "!");
+    }
+    for (int i = 0; i < args.length; i++) {
+      if (args[i] != null) {
+        Class<?> from = args[i].getClass();
+        Class<?> to = types[i];
+        if (!to.isAssignableFrom(from) && !(from.isPrimitive() || to.isPrimitive())) {
+          // Assume autoboxing will work.
+          throw new IllegalArgumentException("Incorrect argument type for param " + i + ": expected " + types[i] + "!");
+        }
+      }
+    }
+    if (shardArrayParamIndex > 0) {
+      if (shardArrayParamIndex >= types.length) {
+        throw new IllegalArgumentException("shardArrayParamIndex out of range!");
+      }
+      if (!(types[shardArrayParamIndex]).equals(String[].class)) {
+        throw new IllegalArgumentException("shardArrayParamIndex parameter (" + shardArrayParamIndex + ") is not of type String[]!");
+      }
+    }
+    if (nodeShardsMap == null || nodeShardsMap.isEmpty()) {
+      return results; // TODO: throw ex?
+    }
+    /*
+     * Make RPC calls to all nodes in parallel.
+     */
     long start = 0;
     if (LOG.isDebugEnabled()) {
       start = System.currentTimeMillis();
     }
-    result.sort(count);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Time for sorting: " + (System.currentTimeMillis() - start) + " ms");
+    ExecutorService executor = Executors.newCachedThreadPool(); // TODO: if 1 node, just call. No thread overhead.
+    Collection<Future<T>> futures = new ArrayList<Future<T>>();
+    for (Map.Entry<String, List<String>> e : nodeShardsMap.entrySet()) {
+      String nodeName = e.getKey();
+      Callable<T> interaction = new NodeInteraction<T>(method, args, shardArrayParamIndex, nodeName, nodeShardsMap);
+      futures.add(executor.submit(interaction));
     }
-    _queryCount++;
-    return result;
-  }
-
-  @Deprecated
-  public int count(final IQuery query, final String[] indexNames) throws KattaException {
+    executor.shutdown();
     try {
-      final QueryParser luceneQueryParser = new QueryParser("field", new KeywordAnalyzer());
-      Query luceneQuery = luceneQueryParser.parse(query.getQuery());
-      return count(luceneQuery, indexNames);
-    } catch (ParseException e) {
-      throw new KattaException("Unable to parse Query: " + query.getQuery(), e);
+      executor.awaitTermination(10000000, TimeUnit.MILLISECONDS); // TODO: config, 10 sec?
+    } catch (InterruptedException e) {
+      LOG.warn("Interrupted while waiting on RPC threads", e);
     }
-  }
-
-    public int count(final Query query, final String[] indexNames) throws KattaException {
-    final Map<String, List<String>> nodeShardsMap = getNode2ShardsMap(indexNames);
-    final List<Integer> result = new ArrayList<Integer>();
-    List<NodeInteraction> nodeInteractions = new ArrayList<NodeInteraction>();
-    for (final String node : nodeShardsMap.keySet()) {
-      nodeInteractions.add(new GetCountInteraction(node, nodeShardsMap, query, result));
+    executor.shutdownNow();
+    /*
+     * Gather the results.
+     */
+    for (Future<T> future : futures) {
+      if (future.isDone()) {
+        try {
+          T result = future.get(0, TimeUnit.MILLISECONDS);
+          if (result != null) {
+            results.add(result);
+          }
+        } catch (ExecutionException e) {
+          /*
+           *  This RPC call threw an exception.
+           *  Stop processing and throw a KattaException.
+           */
+          Throwable t = e.getCause();
+          if (t instanceof KattaException) {
+            // Throw the same KattaException that the RPC call threw.
+            throw (KattaException) t;
+          } else {
+            // Wrap RCP call's exception in a KattaException.
+            throw new KattaException("Error in broadcast()", t);
+          }
+        } catch (TimeoutException e) {
+          /*
+           * Result is not ready. Should not happen, because future is done.
+           * Continue as if RPC call had returned null.
+           */
+          LOG.warn("Timed out while getting RPC result", e);
+        } catch (InterruptedException e) {
+          /*
+           *  Something went wrong while waiting for result. Should not happen
+           *  because we wait for 0 msec, and the future is done. Continue as if
+           *  the RPC call had returned null.
+           */
+          LOG.warn("Interrupted while getting RPC result", e);
+        }
+      }
     }
-    execute(nodeInteractions);
-
-    int resultCount = 0;
-    for (final Integer count : result) {
-      resultCount += count.intValue();
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(String.format("broadcast(%s(%s), %s) took %d msec.",
+              method.getName(), args, nodeShardsMap, (System.currentTimeMillis() - start)));
     }
-    return resultCount;
+    return results;
   }
+  
+  // -------------------- Node management --------------------
 
-  public float getQueryPerMinute() {
-    long time = (System.currentTimeMillis() - _start) / (60 * 1000);
-    time = Math.max(time, 1);
-    return (float) _queryCount / time;
-  }
-
   private Map<String, List<String>> getNode2ShardsMap(final String[] indexNames) throws ShardAccessException {
     String[] indexesToSearchIn = indexNames;
     for (String indexName : indexNames) {
@@ -280,65 +345,39 @@
     return nodeShardsMap;
   }
 
-  public void close() {
-    if (_zkClient != null) {
-      _zkClient.close();
-      Collection<ISearch> proxies = _node2SearchProxyMap.values();
-      for (ISearch search : proxies) {
-        RPC.stopProxy(search);
-      }
-    }
-  }
-
   private List<String> getShardsToSearchIn(String[] indexNames) {
     List<String> shards = new ArrayList<String>();
     for (String index : indexNames) {
-      shards.addAll(_indexToShards.get(index));
+      List<String> theseShards = _indexToShards.get(index);
+      if (theseShards != null) {
+        shards.addAll(theseShards);
+      } else {
+        LOG.warn("No shards found for index " + index);
+      }
     }
     return shards;
   }
 
-  private DocumentFrequenceWritable getDocFrequencies(final Query query, final Map<String, List<String>> node2ShardsMap)
-          throws KattaException {
-    DocumentFrequenceWritable docFreqs = new DocumentFrequenceWritable();
-    List<NodeInteraction> nodeInteractions = new ArrayList<NodeInteraction>();
-    for (final String node : node2ShardsMap.keySet()) {
-      nodeInteractions.add(new GetDocumentFrequencyInteraction(node, node2ShardsMap, query, docFreqs));
+  public double getQueryPerMinute() {
+    double minutes = (System.currentTimeMillis() - _start) / 60000.0;
+    if (minutes > 0.0F) {
+      return _queryCount / minutes;
+    } else {
+      return 0.0F;
     }
-
-    execute(nodeInteractions);
-    return docFreqs;
   }
 
-  private void execute(List<NodeInteraction> nodeInteractions) throws KattaException {
-    long start = 0;
-    if (LOG.isDebugEnabled()) {
-      start = System.currentTimeMillis();
-    }
-    final List<Thread> interactionThreads = new ArrayList<Thread>(nodeInteractions.size());
-    for (NodeInteraction nodeInteraction : nodeInteractions) {
-      final Thread interactionThread = new Thread(nodeInteraction);
-      interactionThreads.add(interactionThread);
-      interactionThread.start();
-      // TODO jz: use thread pool / Executor
-    }
-
-    try {
-      for (final Thread thread : interactionThreads) {
-        thread.join();
+  public void close() {
+    if (_zkClient != null) {
+      _zkClient.close();
+      Collection<VersionedProtocol> proxies = _node2ProxyMap.values();
+      for (VersionedProtocol search : proxies) {
+        RPC.stopProxy(search);
       }
-    } catch (final InterruptedException e) {
-      LOG.warn("Join for search threads interrupted.", e);
     }
-    for (NodeInteraction nodeInteraction : nodeInteractions) {
-      nodeInteraction.checkSuccess();
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug(nodeInteractions.get(0).getClass().getSimpleName() + " took " + (System.currentTimeMillis() - start)
-              + " ms");
-    }
   }
 
+
   protected class IndexStateListener implements IZkDataListener<IndexMetaData> {
 
     public void handleDataAdded(String dataPath, IndexMetaData data) throws KattaException {
@@ -372,61 +411,10 @@
       List<String> removedIndexes = CollectionUtil.getListOfRemoved(indexes, currentIndexes);
       removeIndexes(removedIndexes);
     }
-  }
-
-  public MapWritable getDetails(final Hit hit) throws KattaException {
-    return getDetails(hit, null);
-  }
-
-  public MapWritable getDetails(final Hit hit, final String[] fields) throws KattaException {
-    Map<String, List<String>> node2ShardMap;
-    String node = hit.getNode();
-    List<String> shards = Arrays.asList(hit.getShard());
-    if (_node2SearchProxyMap.containsKey(node)) {
-      node2ShardMap = new HashMap<String, List<String>>(1);
-      node2ShardMap.put(node, shards);
-    } else {
-      node2ShardMap = _selectionPolicy.createNode2ShardsMap(shards);
-      node = node2ShardMap.keySet().iterator().next();
-    }
-
-    GetDetailsInteraction getDetailsInteraction = new GetDetailsInteraction(node, node2ShardMap, hit.getDocId(), fields);
-    getDetailsInteraction.run();
-    getDetailsInteraction.checkSuccess();
-    return getDetailsInteraction.getDetails();
-  }
-
-  public List<MapWritable> getDetails(List<Hit> hits) throws KattaException, InterruptedException {
-    return getDetails(hits, null);
-  }
-
-  public List<MapWritable> getDetails(List<Hit> hits, final String[] fields) throws KattaException, InterruptedException {
-    ExecutorService executorService = Executors.newFixedThreadPool(Math.min(10, hits.size() + 1));
-    List<MapWritable> results = new ArrayList<MapWritable>();
-    List<Future<MapWritable>> futures = new ArrayList<Future<MapWritable>>();
-    for (final Hit hit : hits) {
-      futures.add(executorService.submit(new Callable<MapWritable>() {
-
-        @Override
-        public MapWritable call() throws Exception {
-          return getDetails(hit, fields);
-        }
-      }));
-    }
-
-    for (Future<MapWritable> future : futures) {
-      try {
-        results.add(future.get());
-      } catch (ExecutionException e) {
-        throw new KattaException("Could not get hit details.", e.getCause());
-      }
-    }
-
-    executorService.shutdown();
     
-    return results;
   }
 
+
   protected class ShardNodeListener implements IZkChildListener {
 
     public void handleChildChange(String parentPath, List<String> currentNodes) throws KattaException {
@@ -436,101 +424,10 @@
       // update shard2Nodes mapping
       updateSelectionPolicy(shardName, currentNodes);
     }
+    
   }
 
-  private class GetDocumentFrequencyInteraction extends NodeInteraction {
 
-    private final Query _query;
-    private final DocumentFrequenceWritable _docFreqs;
-
-    public GetDocumentFrequencyInteraction(String node, Map<String, List<String>> node2ShardsMap, Query query,
-            DocumentFrequenceWritable docFreqs) {
-      super(node, node2ShardsMap);
-      _query = query;
-      _docFreqs = docFreqs;
-    }
-
-    @Override
-    protected void doInteraction(ISearch search, String node, List<String> shards) throws IOException {
-      final DocumentFrequenceWritable nodeDocFreqs = search.getDocFreqs(new QueryWritable(_query), shards
-              .toArray(new String[shards.size()]));
-      _docFreqs.addNumDocs(nodeDocFreqs.getNumDocs());
-      _docFreqs.putAll(nodeDocFreqs.getAll());
-    }
-  }
-
-  private class GetCountInteraction extends NodeInteraction {
-
-    private final Query _query;
-    private final List<Integer> _result;
-
-    public GetCountInteraction(String node, Map<String, List<String>> node2ShardsMap, Query query, List<Integer> result) {
-      super(node, node2ShardsMap);
-      _query = query;
-      _result = result;
-    }
-
-    @Override
-    protected void doInteraction(ISearch search, String node, List<String> shards) throws IOException {
-      final int count = search.getResultCount(new QueryWritable(_query), shards.toArray(new String[shards.size()]));
-      _result.add(count);
-    }
-  }
-
-  private class GetDetailsInteraction extends NodeInteraction {
-
-    private final int _docId;
-    private final String[] _fields;
-    private MapWritable _details;
-
-    public GetDetailsInteraction(String node, Map<String, List<String>> node2ShardsMap, int docId, String[] fields) {
-      super(node, node2ShardsMap);
-      _docId = docId;
-      _fields = fields;
-    }
-
-    @Override
-    protected void doInteraction(ISearch search, String node, List<String> shards) throws IOException {
-      String shard = shards.get(0);
-      if (_fields == null) {
-        _details = search.getDetails(shard, _docId);
-      } else {
-        _details = search.getDetails(shard, _docId, _fields);
-      }
-    }
-
-    public MapWritable getDetails() {
-      return _details;
-    }
-  }
-
-  private class SearchInteraction extends NodeInteraction {
-
-    private final Query _query;
-    private final int _count;
-    private final DocumentFrequenceWritable _docFreqs;
-    private final Hits _result;
-
-    public SearchInteraction(String node, Map<String, List<String>> node2ShardsMap, Query query,
-            DocumentFrequenceWritable docFreqs, Hits result, int count) {
-      super(node, node2ShardsMap);
-      _query = query;
-      _docFreqs = docFreqs;
-      _result = result;
-      _count = count;
-    }
-
-    @Override
-    protected void doInteraction(ISearch search, String node, List<String> shards) throws IOException {
-      Hits hits;
-      final String[] shardsArray = shards.toArray(new String[shards.size()]);
-      final HitsMapWritable shardToHits = search.search(new QueryWritable(_query), _docFreqs, shardsArray, _count);
-      hits = shardToHits.getHits();
-      _result.addHits(hits.getHits());
-      _result.addTotalHits(hits.size());
-    }
-  }
-
   /**
    * This class encapsulates an interaction with one or multiple node's in order
    * to query information for a set of shards.
@@ -538,78 +435,82 @@
    * Given the fact that shards a replicated about nodes, this class tries node
    * after node to get the desired information.
    */
-  private abstract class NodeInteraction implements Runnable {
+  private class NodeInteraction<T> implements Callable<T> {
 
+    private final Method _method;
+    private final Object[] _args;
+    private final int _shardArrayIndex;
     private final String _node;
     private final Map<String, List<String>> _node2ShardsMap;
     private final List<String> _triedNodes = new ArrayList<String>(1);
     private int _tries = 0;
-    private Exception _exception;
+    private T result;
 
-    public NodeInteraction(String node, Map<String, List<String>> node2ShardsMap) {
+    public NodeInteraction(Method method, Object[] args, int shardArrayIndex, String node, Map<String, List<String>> node2ShardsMap) {
+      _method = method;
+      // Make a copy if we will be modifying this.
+      _args = shardArrayIndex > 0 ? Arrays.copyOf(args, args.length) : args;
+      _shardArrayIndex = shardArrayIndex;
       _node = node;
       _node2ShardsMap = node2ShardsMap;
     }
 
-    public final void run() {
-      interact(_node, _node2ShardsMap);
+    public T call() throws Exception {
+      return interact(_node, _node2ShardsMap);
     }
-
-    protected final void interact(String node, Map<String, List<String>> node2ShardsMap) {
+    
+    @SuppressWarnings("unchecked")
+    public T interact(String node, Map<String, List<String>> node2ShardsMap) throws Exception {
       List<String> shards = node2ShardsMap.get(node);
+      _tries++;
+      _triedNodes.add(node);
+      VersionedProtocol proxy = _node2ProxyMap.get(node);
       try {
-        _tries++;
-        _triedNodes.add(node);
-        ISearch searcher = _node2SearchProxyMap.get(node);
-        try {
-          if (searcher == null) {
-            throw new IOException("node proxy for node " + node + " is not available any more");
-          }
-          long startTime = 0;
-          if (LOG.isDebugEnabled()) {
-            startTime = System.currentTimeMillis();
-          }
-          doInteraction(searcher, node, shards);
-          if (LOG.isDebugEnabled()) {
-            LOG.debug(getClass().getSimpleName() + " with node " + node + " took "
-                    + (System.currentTimeMillis() - startTime) + " ms.");
-          }
-        } catch (IOException e) {
-          if (_tries == 3) {
-            throw new KattaException(getClass().getSimpleName() + " for shards " + shards + " failed. Tried nodes: "
-                    + _triedNodes);
-          }
-          LOG.warn(
-                  "failed to interact with node " + node + ". Try with other node(s) " + node2ShardsMap.keySet() + ".",
-                  e);
-          Map<String, List<String>> node2ShardsMapForFailedNode = prepareRetry(node, shards);
+        if (proxy == null) {
+          throw new IOException("Node proxy for node " + node + " is not available any more");
+        }
+        long startTime = 0;
+        if (_shardArrayIndex >= 0) {
+          // We need to pass the list of shards to the server's method.
+          _args[_shardArrayIndex] = shards.toArray(new String[shards.size()]);
+        }
+        String methodDesc = null;
+        if (LOG.isDebugEnabled()) {
+          methodDesc = describeMethodCall(_method, _args, node);
+          LOG.debug("About to invoke " + methodDesc);
+          startTime = System.currentTimeMillis();
+        }
+        result = (T) _method.invoke(proxy, _args);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Calling " + methodDesc + " took " + (System.currentTimeMillis() - startTime) + " msec");
+        }
+        return result;
+      } catch (Throwable e) {
+        if (_tries == 3) {
+          throw new KattaException(getClass().getSimpleName() + " for shards " + shards + " failed. Tried nodes: "
+                  + _triedNodes);
+        }
+        LOG.warn(
+                "failed to interact with node " + node + ". Try with other node(s) " + node2ShardsMap.keySet() + ".",
+                e);
+        Map<String, List<String>> node2ShardsMapForFailedNode = prepareRetry(node, shards);
 
-          // execute the action again for every node
-          for (String newNode : node2ShardsMapForFailedNode.keySet()) {
-            // TODO jz: if more then one node we should spawn new
-            // threads
-            interact(newNode, node2ShardsMapForFailedNode);
+        // Execute the action again for every node
+        for (String newNode : node2ShardsMapForFailedNode.keySet()) {
+          // TODO jz: if more then one node we should spawn new threads.
+          T result = interact(newNode, node2ShardsMapForFailedNode);
+          if (result != null) {  // TODO: is this ok?
+            return result;
           }
         }
-      } catch (Exception e) {
-        _exception = e;
+        return null;
       }
     }
 
-    public void checkSuccess() throws KattaException {
-      if (_exception != null) {
-        if (_exception instanceof KattaException) {
-          throw (KattaException) _exception;
-        }
-        throw new KattaException(getClass().getSimpleName() + " for shards " + _node2ShardsMap.get(_node)
-                + " failed. Tried nodes: " + _triedNodes, _exception);
-      }
-    }
-
     private Map<String, List<String>> prepareRetry(String node, List<String> shards) throws ShardAccessException {
       // remove node
       _node2ShardsMap.remove(node);
-      _node2SearchProxyMap.remove(node);
+      _node2ProxyMap.remove(node);
       _selectionPolicy.removeNode(node);
 
       // find new node(s) for the shards and add to global node2ShardMap
@@ -625,7 +526,40 @@
       return node2ShardsMapForFailedNode;
     }
 
-    protected abstract void doInteraction(ISearch search, String node, List<String> shards) throws IOException;
+    public Object getResult() {
+      return result;
+    }
+    
+    private String describeMethodCall(Method method, Object[] args, String nodeName) {
+      StringBuffer buf = new StringBuffer(method.getName());
+      buf.append("(");
+      String sep = "";
+      for (int i=0; i<args.length; i++) {
+        buf.append(sep);
+        if (args[i] == null) {
+          buf.append("null");
+        } else if (args[i] instanceof String[]) {
+          String[] strs = (String[]) args[i];
+          String sep2 = "";
+          buf.append("[");
+          for (String str : strs) {
+            buf.append(sep2);
+            buf.append("\"");
+            buf.append(str);
+            buf.append("\"");
+            sep2 = ", ";
+          }
+          buf.append("]");
+        } else {
+          buf.append(args[i].toString());
+        }
+        sep = ", ";
+      }
+      buf.append(") on ");
+      buf.append(nodeName);
+      return buf.toString();
+    }
+    
   }
 
 }

Property changes on: bin
___________________________________________________________________
Name: svn:ignore
   + zookeeper-log-data
zookeeper-data



Index: pom.xml
===================================================================
--- pom.xml	(.../tags/ajohn/pristine_0_5_1/katta)	(revision 0)
+++ pom.xml	(.../trunk/katta)	(revision 10997)
@@ -0,0 +1,269 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.deepdyve</groupId>
+  <artifactId>katta</artifactId>
+  <packaging>jar</packaging>
+  <version>0.5.2-DEEPDYVE-SNAPSHOT</version>
+  <name></name>
+  <url>http://maven.apache.org</url>
+  
+  <dependencies>
+    <dependency>
+	<groupId>org.hamcrest</groupId>
+	<artifactId>hamcrest-all</artifactId>
+	<version>1.1</version>
+    </dependency> 
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>1.7</version>
+    </dependency> 
+<dependency>
+  <groupId>org.jmock</groupId>
+  <artifactId>jmock-junit3</artifactId>
+  <version>2.5.1</version>
+</dependency>
+    <!--<dependency>
+      <groupId>jmock</groupId>
+      <artifactId>jmock</artifactId>
+      <version>2.4.0</version>
+    </dependency>-->
+    <dependency>
+      <groupId>com.deepdyve.ddcore</groupId>
+      <artifactId>ddcore</artifactId>
+      <version>1.2.1-SNAPSHOT</version>
+    </dependency>
+    <dependency>
+      <groupId>ddlogging</groupId>
+      <artifactId>ddlogging</artifactId>
+      <version>1.0</version>
+    </dependency>
+    <dependency>
+      <groupId>activation</groupId>
+      <artifactId>activation</artifactId>
+      <version>1.0</version>
+    </dependency>
+    <dependency>
+      <groupId>antlr</groupId>
+      <artifactId>antlr</artifactId>
+      <version>2.7.6</version>
+    </dependency>
+    <dependency>
+      <groupId>log4j</groupId>
+      <artifactId>log4j</artifactId>
+      <version>1.2.14</version>
+    </dependency>
+    <dependency>
+      <groupId>apache-solr</groupId>
+      <artifactId>apache-solr</artifactId>
+      <version>1.2.0</version>
+    </dependency>
+    <dependency>
+      <groupId>asm</groupId>
+      <artifactId>asm</artifactId>
+      <version>1.0</version>
+    </dependency>
+    <dependency>
+      <groupId>cglib</groupId>
+      <artifactId>cglib</artifactId>
+      <version>2.1.3</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-cli</groupId>
+      <artifactId>commons-cli</artifactId>
+      <version>2.0-SNAPSHOT</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+      <version>1.3</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-collections</groupId>
+      <artifactId>commons-collections</artifactId>
+      <version>2.1.1</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-httpclient</groupId>
+      <artifactId>commons-httpclient</artifactId>
+      <version>3.1</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-logging</groupId>
+      <artifactId>commons-logging</artifactId>
+      <version>1.1</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-logging-api</groupId>
+      <artifactId>commons-logging-api</artifactId>
+      <version>1.0.4</version>
+    </dependency>
+    <dependency>
+      <groupId>dom4j</groupId>
+      <artifactId>dom4j</artifactId>
+      <version>1.6.1</version>
+    </dependency>
+    <dependency>
+      <groupId>easymock</groupId>
+      <artifactId>easymock</artifactId>
+      <version>1.0</version>
+    </dependency>
+    <dependency>
+      <groupId>gnujaxp</groupId>
+      <artifactId>gnujaxp</artifactId>
+      <version>1.0</version>
+    </dependency>
+    <dependency>
+      <groupId>itext</groupId>
+      <artifactId>itext</artifactId>
+      <version>2.0.2</version>
+    </dependency>
+    <dependency>
+      <groupId>jcommon</groupId>
+      <artifactId>jcommon</artifactId>
+      <version>1.0.10</version>
+    </dependency>
+    <dependency>
+      <groupId>jets3t</groupId>
+      <artifactId>jets3t</artifactId>
+      <version>0.6.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.mortbay.jetty</groupId>
+      <artifactId>jetty</artifactId>
+      <version>6.1.7</version>
+    </dependency>
+    <dependency>
+      <groupId>org.mortbay.jetty</groupId>
+      <artifactId>jetty-util</artifactId>
+      <version>6.1.7</version>
+    </dependency>
+    <dependency>
+      <groupId>json-simple</groupId>
+      <artifactId>json-simple</artifactId>
+      <version>1.0.2</version>
+    </dependency>
+    <dependency>
+      <groupId>jta</groupId>
+      <artifactId>jta</artifactId>
+      <version>1.0</version>
+    </dependency>
+    <dependency>
+      <groupId>lucene-core</groupId>
+      <artifactId>lucene-core</artifactId>
+      <version>2.4.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.mortbay.jetty</groupId>
+      <artifactId>servlet-api-2.5</artifactId>
+      <version>6.1.7</version>
+    </dependency>
+    <dependency>
+      <groupId>xercesImpl</groupId>
+      <artifactId>xercesImpl</artifactId>
+      <version>2.7.1</version>
+    </dependency>
+    <dependency>
+      <groupId>xml-apis</groupId>
+      <artifactId>xml-apis</artifactId>
+      <version>1.0</version>
+    </dependency>
+    <dependency>
+      <groupId>xmlenc</groupId>
+      <artifactId>xmlenc</artifactId>
+      <version>0.52</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache</groupId>
+      <artifactId>zookeeper</artifactId>
+      <version>3.1.0</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.5</version>
+      <scope>test</scope>    
+    </dependency>
+  </dependencies>
+  
+  <build>
+    <resources>
+	    <resource>
+	      <directory>bin</directory>
+	    </resource>
+	    <resource>
+	      <directory>conf</directory>
+	    </resource>
+	  </resources>  	
+  	
+    <plugins>
+ 
+ 
+     <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <source>1.6</source>
+          <target>1.6</target>
+          <encoding>UTF-8</encoding>
+        </configuration>
+      </plugin>
+
+       <plugin>
+          <groupId>org.codehaus.mojo</groupId>
+          <artifactId>build-helper-maven-plugin</artifactId>
+          <executions>
+            <execution>
+              <id>add-source</id>
+              <phase>generate-sources</phase>
+              <goals>
+                <goal>add-source</goal>
+              </goals>
+              <configuration>
+                <sources>
+                    <source>${basedir}/gen-java</source>
+                    <source>${basedir}/src/main/resources</source>
+                </sources>
+              </configuration>
+            </execution>
+          </executions>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-source-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>attach-sources</id>
+            <goals>
+              <goal>jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+
+    </plugins>
+  </build>
+  <distributionManagement>
+    <snapshotRepository>
+     <uniqueVersion>true</uniqueVersion>
+     <id>libs-snapshots-local</id>
+     <name>libs-snapshots-local</name>
+     <url>http://dev.sjc.infovell.com/artifactory/libs-snapshots-local</url>
+    </snapshotRepository>
+    <repository>
+      <id>repo</id>
+      <name>katta</name>
+      <url>file://${basedir}/.m2/repository/</url>
+    </repository>
+  </distributionManagement>
+
+  <repositories>
+    <repository>
+      <id>DeepDyve-releases</id>
+      <name>DeepDyve-releases</name>
+      <url>http://dev.sjc.infovell.com/artifactory/repo</url>
+    </repository>
+  </repositories>
+</project>

Property changes on: .
___________________________________________________________________
Name: svn:ignore
   + .project
.classpath
.settings
build
target