From 1eb422623030ef8e2452950f37987c4b3c7765a7 Mon Sep 17 00:00:00 2001
From: Andrew Bayer <abayer@apache.org>
Date: Fri, 22 Jul 2011 20:03:46 +0000
Subject: [PATCH] SIP-3. File format for large object (LOB) storage.

Introduce LobFile format for storing large objects.
Implemented LobFile.Reader, LobFile.Writer classes.
Added a performance test of LobFile reading/writing speed.
Build system: fix cobertura build deps.
Remove unused utility classes from o.a.h.s.io.
Use LobFile for external storage in {B,C}lobRef.
Added LobReaderCache.
Converted BlobRef to read from LobFiles (through LobReaderCache).
LargeObjectLoader writes to LobFiles.
Common code from BlobRef and ClobRef factored out into LobRef abstract
base class.
Updated Test{B,C}lobRef and TestLargeObjectLoader for new external LOB storage.
Updated *ImportMappers to close LargeObjectLoaders when they're done.
Added performance tests to build.
Added script to run perf tests; factored out common logic into config script.
Fixed ivy dependency resolution to use multiple configuration inheritance.
Added LobFileStressTest.
Added readme with instructions to src/perftest directory.
Added CodecMap that abstracts compression codec classes to names.

From: Aaron Kimball <aaron@cloudera.com>

git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149897 13f79535-47bb-0310-9956-ffa450edef68
---
 bin/configure-sqoop                           |   72 +
 bin/sqoop                                     |   43 +-
 build.xml                                     |   32 +-
 ivy.xml                                       |   10 +-
 .../org/apache/hadoop/sqoop/io/CodecMap.java  |   99 +
 .../hadoop/sqoop/io/CountingOutputStream.java |   74 -
 .../sqoop/io/FixedLengthInputStream.java      |   90 +
 .../sqoop/io/HdfsSplitOutputStream.java       |  154 --
 .../org/apache/hadoop/sqoop/io/LobFile.java   | 1802 +++++++++++++++++
 .../hadoop/sqoop/io/LobReaderCache.java       |  146 ++
 .../sqoop/io/UnsupportedCodecException.java   |   38 +
 .../org/apache/hadoop/sqoop/lib/BlobRef.java  |  160 +-
 .../org/apache/hadoop/sqoop/lib/ClobRef.java  |  158 +-
 .../hadoop/sqoop/lib/LargeObjectLoader.java   |  145 +-
 .../org/apache/hadoop/sqoop/lib/LobRef.java   |  295 +++
 .../mapreduce/SequenceFileImportMapper.java   |   19 +-
 .../sqoop/mapreduce/TextImportMapper.java     |   18 +-
 src/perftest/LobFilePerfTest.java             |  112 +
 src/perftest/LobFileStressTest.java           |  391 ++++
 src/perftest/README-perftest.txt              |   40 +
 src/scripts/run-perftest.sh                   |   39 +
 .../org/apache/hadoop/sqoop/SmokeTests.java   |    2 +
 .../apache/hadoop/sqoop/io/TestLobFile.java   |  577 ++++++
 .../apache/hadoop/sqoop/lib/TestBlobRef.java  |   33 +-
 .../apache/hadoop/sqoop/lib/TestClobRef.java  |   44 +-
 .../sqoop/lib/TestLargeObjectLoader.java      |    4 +-
 26 files changed, 4012 insertions(+), 585 deletions(-)
 create mode 100755 bin/configure-sqoop
 create mode 100644 src/java/org/apache/hadoop/sqoop/io/CodecMap.java
 delete mode 100644 src/java/org/apache/hadoop/sqoop/io/CountingOutputStream.java
 create mode 100644 src/java/org/apache/hadoop/sqoop/io/FixedLengthInputStream.java
 delete mode 100644 src/java/org/apache/hadoop/sqoop/io/HdfsSplitOutputStream.java
 create mode 100644 src/java/org/apache/hadoop/sqoop/io/LobFile.java
 create mode 100644 src/java/org/apache/hadoop/sqoop/io/LobReaderCache.java
 create mode 100644 src/java/org/apache/hadoop/sqoop/io/UnsupportedCodecException.java
 create mode 100644 src/java/org/apache/hadoop/sqoop/lib/LobRef.java
 create mode 100644 src/perftest/LobFilePerfTest.java
 create mode 100644 src/perftest/LobFileStressTest.java
 create mode 100644 src/perftest/README-perftest.txt
 create mode 100755 src/scripts/run-perftest.sh
 create mode 100644 src/test/org/apache/hadoop/sqoop/io/TestLobFile.java

diff --git a/bin/configure-sqoop b/bin/configure-sqoop
new file mode 100755
index 00000000..fef48e86
--- /dev/null
+++ b/bin/configure-sqoop
@@ -0,0 +1,72 @@
+#!/bin/sh
+#
+# Licensed to Cloudera, Inc. under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is sourced in by bin/sqoop to set environment variables prior to
+# invoking Hadoop.
+
+prgm=`readlink -f $0`
+bin=`dirname ${prgm}`
+bin=`cd ${bin} && pwd`
+
+if [ -z "$SQOOP_HOME" ]; then
+  export SQOOP_HOME=${bin}/..
+fi
+
+if [ -z "${HADOOP_HOME}" ]; then
+  # Try CDH default if the user hasn't set this.
+  HADOOP_HOME=/usr/lib/hadoop
+fi
+
+# Where to find the main Sqoop jar
+SQOOP_JAR_DIR=$SQOOP_HOME
+
+# Where to find the shim jars.
+SQOOP_SHIM_DIR=$SQOOP_HOME/shims
+
+# If there's a "build" subdir, override with this, so we use
+# the newly-compiled copy.
+if [ -d "$SQOOP_JAR_DIR/build" ]; then
+  SQOOP_JAR_DIR="${SQOOP_JAR_DIR}/build"
+
+  if [ -d "$SQOOP_JAR_DIR/shims" ]; then
+    SQOOP_SHIM_DIR="$SQOOP_JAR_DIR/shims"
+  fi
+fi
+
+# Add sqoop dependencies to classpath.
+SQOOP_CLASSPATH=""
+if [ -d "$SQOOP_HOME/lib" ]; then
+  for f in $SQOOP_HOME/lib/*.jar; do
+    SQOOP_CLASSPATH=${SQOOP_CLASSPATH}:$f;
+  done
+fi
+
+# If there's a build subdir, use Ivy-retrieved dependencies too.
+if [ -d "$SQOOP_HOME/build/ivy/lib/sqoop" ]; then
+  for f in $SQOOP_HOME/build/ivy/lib/sqoop/*/*.jar; do
+    SQOOP_CLASSPATH=${SQOOP_CLASSPATH}:$f;
+  done
+fi
+
+export SQOOP_CLASSPATH
+export SQOOP_JAR_DIR
+export SQOOP_SHIM_DIR
+export SQOOP_JAR=`ls -1 ${SQOOP_JAR_DIR}/sqoop-*.jar | head -n 1`
+export HADOOP_CLASSPATH="${SQOOP_CLASSPATH}:${HADOOP_CLASSPATH}"
+export HADOOP_HOME
+export HADOOP_OPTS="-Dsqoop.shim.jar.dir=${SQOOP_SHIM_DIR} ${HADOOP_OPTS}"
+
diff --git a/bin/sqoop b/bin/sqoop
index 7bdf5b06..8a8f0118 100755
--- a/bin/sqoop
+++ b/bin/sqoop
@@ -19,43 +19,6 @@ prgm=`readlink -f $0`
 bin=`dirname ${prgm}`
 bin=`cd ${bin} && pwd`
 
-if [ -z "$SQOOP_HOME" ]; then
-  SQOOP_HOME=${bin}/..
-fi
-
-if [ -z "${HADOOP_HOME}" ]; then
-  # Try CDH default if the user hasn't set this.
-  HADOOP_HOME=/usr/lib/hadoop
-fi
-
-# Where to find the main Sqoop jar
-SQOOP_JAR_DIR=$SQOOP_HOME
-
-# Where to find the shim jars.
-SQOOP_SHIM_DIR=$SQOOP_HOME/shims
-
-# If there's a "build" subdir, override with this, so we use
-# the newly-compiled copy.
-if [ -d "$SQOOP_JAR_DIR/build" ]; then
-  SQOOP_JAR_DIR="${SQOOP_JAR_DIR}/build"
-
-  if [ -d "$SQOOP_JAR_DIR/shims" ]; then
-    SQOOP_SHIM_DIR="$SQOOP_JAR_DIR/shims"
-  fi
-fi
-
-# Add sqoop dependencies to classpath.
-SQOOP_CLASSPATH=""
-if [ -d "$SQOOP_HOME/lib" ]; then
-  for f in $SQOOP_HOME/lib/*.jar; do
-    SQOOP_CLASSPATH=${SQOOP_CLASSPATH}:$f;
-  done
-fi
-
-SQOOP_JAR=`ls -1 ${SQOOP_JAR_DIR}/sqoop-*.jar | head -n 1`
-
-# Invoke Hadoop
-HADOOP_CLASSPATH="${SQOOP_CLASSPATH}:${HADOOP_CLASSPATH}" \
-  HADOOP_OPTS="-Dsqoop.shim.jar.dir=${SQOOP_SHIM_DIR} ${HADOOP_OPTS}" \
-  ${HADOOP_HOME}/bin/hadoop jar ${SQOOP_JAR} org.apache.hadoop.sqoop.Sqoop "$@"
-
+source ${bin}/configure-sqoop
+${HADOOP_HOME}/bin/hadoop jar ${SQOOP_JAR} \
+    org.apache.hadoop.sqoop.Sqoop "$@"
diff --git a/build.xml b/build.xml
index 945b2892..61e8aa53 100644
--- a/build.xml
+++ b/build.xml
@@ -39,6 +39,7 @@
   <property name="src.dir" location="${base.src.dir}/java" />
   <property name="test.dir" location="${base.src.dir}/test" />
   <property name="shim.src.dir" location="${base.src.dir}/shims" />
+  <property name="perftest.src.dir" location="${base.src.dir}/perftest" />
   <property name="lib.dir" location="${basedir}/lib" />
   <property name="docs.src.dir" location="${base.src.dir}/docs" />
   <property name="script.src.dir" location="${base.src.dir}/scripts" />
@@ -50,6 +51,8 @@
   <property name="build.shim.classes" location="${build.shim.dir}/classes"/>
   <property name="build.test" location="${build.dir}/test"/>
   <property name="build.test.classes" location="${build.test}/classes" />
+  <property name="build.perftest" location="${build.dir}/perftest"/>
+  <property name="build.perftest.classes" location="${build.perftest}/classes"/>
   <property name="build.javadoc" location="${build.dir}/docs/api" />
   <property name="test.log.dir" location="${build.dir}/test/logs"/>
   <property name="dist.dir" location="${build.dir}/${artifact.name}" />
@@ -123,7 +126,6 @@
   <path id="compile.classpath">
     <pathelement location="${build.classes}"/>
     <path refid="lib.path"/>
-    <path refid="${name}.common.classpath"/>
     <path refid="${name}.hadoop.classpath"/>
   </path>
 
@@ -131,7 +133,6 @@
   <path id="test.classpath">
     <pathelement location="${build.test.classes}" />
     <path refid="${name}.hadooptest.classpath"/>
-    <path refid="${name}.test.classpath"/>
     <path refid="compile.classpath"/>
   </path>
 
@@ -146,7 +147,7 @@
 
   <!-- Compile a shim class so Sqoop can run with the specified hadoop.dist -->
   <target name="compile-one-shim"
-      depends="init, ivy-retrieve-common, ivy-retrieve-hadoop">
+      depends="init, ivy-retrieve-hadoop">
     <mkdir dir="${build.shim.classes}/${hadoop.dist}" />
     <javac
         encoding="${build.encoding}"
@@ -178,7 +179,7 @@
   
   <!-- Compile core classes for the project -->
   <target name="compile"
-      depends="init, ivy-retrieve-common, ivy-retrieve-hadoop"
+      depends="init, ivy-retrieve-hadoop"
       description="Compile core classes for the project">
     <!-- don't use an out-of-date instrumented build. -->
     <delete dir="${cobertura.class.dir}" />
@@ -196,7 +197,7 @@
   </target>
 
   <target name="compile-test"
-      depends="compile, ivy-retrieve-test, ivy-retrieve-hadoop-test"
+      depends="compile, ivy-retrieve-hadoop-test"
       description="Compile test classes">
     <mkdir dir="${build.test.classes}" />
     <javac
@@ -211,12 +212,29 @@
     </javac>
   </target>
 
+  <target name="compile-perf-test"
+      depends="compile, ivy-retrieve-hadoop-test"
+      description="Compile manual performance tests">
+    <mkdir dir="${build.perftest.classes}" />
+    <javac
+        encoding="${build.encoding}"
+        srcdir="${perftest.src.dir}"
+        includes="**/*.java"
+        destdir="${build.perftest.classes}"
+        debug="${javac.debug}">
+      <classpath>
+        <path refid="test.classpath"/>
+      </classpath>
+    </javac>
+  </target>
+
   <target name="jar" depends="compile" description="Create main jar">
     <jar jarfile="${build.dir}/${dest.jar}" basedir="${build.classes}" />
   </target>
 
   <!-- Ensure that all source code can be built -->
-  <target name="compile-all" depends="compile,compile-test,jar-all-shims"
+  <target name="compile-all"
+      depends="compile,compile-test,jar-all-shims,compile-perf-test"
       description="Compile all sources"/>
 
   <target name="scripts" depends="jar"
@@ -542,7 +560,7 @@
   </target>
 
   <target name="cobertura"
-      depends="check-for-cobertura,warn-cobertura-unset,jar,compile-test,test-prep"
+      depends="check-for-cobertura,warn-cobertura-unset,jar,compile-test,test-prep,jar-all-shims"
       if="cobertura.present" description="Run Cobertura (code coverage)">
     <taskdef classpathref="cobertura.classpath"
         resource="tasks.properties"/>
diff --git a/ivy.xml b/ivy.xml
index 0c657f71..a10d1f1e 100644
--- a/ivy.xml
+++ b/ivy.xml
@@ -33,18 +33,18 @@
       extends="runtime"
       description="artifacts needed to compile/test the application"/>
     <conf name="apache" visibility="private"
-      extends="runtime"
+      extends="common,runtime"
       description="artifacts from Apache for compile/test" />
     <conf name="cloudera" visibility="private"
-      extends="runtime"
+      extends="common,runtime"
       description="artifacts from Cloudera for compile/test" />
 
-    <conf name="test" visibility="private" extends="runtime"/>
+    <conf name="test" visibility="private" extends="common,runtime"/>
     <conf name="apachetest" visibility="private"
-      extends="test"
+      extends="test,apache"
       description="artifacts from Apache for testing" />
     <conf name="clouderatest" visibility="private"
-      extends="test"
+      extends="test,cloudera"
       description="artifacts from Cloudera for testing" />
 
     <!-- We don't redistribute everything we depend on (e.g., Hadoop itself);
diff --git a/src/java/org/apache/hadoop/sqoop/io/CodecMap.java b/src/java/org/apache/hadoop/sqoop/io/CodecMap.java
new file mode 100644
index 00000000..d97cc207
--- /dev/null
+++ b/src/java/org/apache/hadoop/sqoop/io/CodecMap.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop.io;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * Provides a mapping from codec names to concrete implementation class names.
+ * This is used by LobFile.
+ */
+public final class CodecMap {
+
+  // Supported codec map values
+  public static final String NONE = "none";
+  public static final String DEFLATE = "deflate";
+  public static final String LZO = "lzo";
+
+  private static Map<String, String> codecNames;
+  static {
+    codecNames = new TreeMap<String, String>();
+
+    // Register the names of codecs we know about.
+    codecNames.put(NONE,    null);
+    codecNames.put(DEFLATE, "org.apache.hadoop.io.compress.DefaultCodec");
+    codecNames.put(LZO,     "com.hadoop.compression.lzo.LzoCodec");
+  }
+
+  private CodecMap() {
+  }
+
+  /**
+   * Given a codec name, return the name of the concrete class
+   * that implements it (or 'null' in the case of the "none" codec).
+   * @throws UnsupportedCodecException if a codec cannot be found
+   * with the supplied name.
+   */
+  public static String getCodecClassName(String codecName)
+      throws UnsupportedCodecException {
+    if (!codecNames.containsKey(codecName)) {
+      throw new UnsupportedCodecException(codecName);
+    }
+
+    return codecNames.get(codecName);
+  }
+
+  /**
+   * Given a codec name, instantiate the concrete implementation
+   * class that implements it.
+   * @throws UnsupportedCodecException if a codec cannot be found
+   * with the supplied name.
+   */
+  public static CompressionCodec getCodec(String codecName,
+      Configuration conf) throws UnsupportedCodecException {
+    String codecClassName = null;
+    try {
+      codecClassName = getCodecClassName(codecName);
+      if (null == codecClassName) {
+        return null;
+      }
+      Class<? extends CompressionCodec> codecClass =
+          (Class<? extends CompressionCodec>)
+          conf.getClassByName(codecClassName);
+      return (CompressionCodec) ReflectionUtils.newInstance(
+          codecClass, conf);
+    } catch (ClassNotFoundException cnfe) {
+      throw new UnsupportedCodecException("Cannot find codec class "
+          + codecClassName + " for codec " + codecName);
+    }
+  }
+
+  /**
+   * Return the set of available codec names.
+   */
+  public static Set<String> getCodecNames() {
+    return codecNames.keySet();
+  }
+}
diff --git a/src/java/org/apache/hadoop/sqoop/io/CountingOutputStream.java b/src/java/org/apache/hadoop/sqoop/io/CountingOutputStream.java
deleted file mode 100644
index c2ba4f39..00000000
--- a/src/java/org/apache/hadoop/sqoop/io/CountingOutputStream.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to Cloudera, Inc. under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  Cloudera, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.sqoop.io;
-
-import java.io.OutputStream;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-/**
- * An output stream that counts how many bytes its written.
- */
-public class CountingOutputStream extends OutputStream {
-
-  public static final Log LOG = LogFactory.getLog(CountingOutputStream.class.getName());
-
-  private final OutputStream stream;
-  private long bytesWritten;
-
-  public CountingOutputStream(final OutputStream outputStream) {
-    this.stream = outputStream;
-    this.bytesWritten = 0;
-  }
-
-  /** @return the number of bytes written thus far to the stream. */
-  public long getBytesWritten() {
-    return bytesWritten;
-  }
-
-  /** Reset the counter of bytes written to zero. */
-  public void resetCount() {
-    this.bytesWritten = 0;
-  }
-
-  public void close() throws IOException {
-    this.stream.close();
-  }
-
-  public void flush() throws IOException {
-    this.stream.flush();
-  }
-
-  public void write(byte [] b) throws IOException {
-    this.stream.write(b);
-    bytesWritten += b.length;
-  }
-
-  public void write(byte [] b, int off, int len) throws IOException {
-    this.stream.write(b, off, len);
-    bytesWritten += len;
-  }
-
-  public void write(int b) throws IOException {
-    this.stream.write(b);
-    bytesWritten++;
-  }
-}
diff --git a/src/java/org/apache/hadoop/sqoop/io/FixedLengthInputStream.java b/src/java/org/apache/hadoop/sqoop/io/FixedLengthInputStream.java
new file mode 100644
index 00000000..d3a9828c
--- /dev/null
+++ b/src/java/org/apache/hadoop/sqoop/io/FixedLengthInputStream.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop.io;
+
+import java.io.FilterInputStream;
+import java.io.InputStream;
+import java.io.IOException;
+
+import org.apache.commons.io.input.CloseShieldInputStream;
+import org.apache.commons.io.input.CountingInputStream;
+import org.apache.commons.io.input.ProxyInputStream;
+
+/**
+ * Provides an InputStream that can consume a fixed maximum number of bytes
+ * from an underlying stream. Closing the FixedLengthInputStream does not
+ * close the underlying stream. After reading the maximum number of available
+ * bytes this acts as though EOF has been reached.
+ */
+public class FixedLengthInputStream extends ProxyInputStream {
+
+  private CountingInputStream countingIn;
+  private long maxBytes;
+
+  public FixedLengthInputStream(InputStream stream, long maxLen) {
+    super(new CountingInputStream(new CloseShieldInputStream(stream)));
+
+    // Save a correctly-typed reference to the underlying stream.
+    this.countingIn = (CountingInputStream) this.in;
+    this.maxBytes = maxLen;
+  }
+
+  /** @return the number of bytes already consumed by the client. */
+  private long consumed() {
+    return countingIn.getByteCount();
+  }
+
+  /**
+   * @return number of bytes remaining to be read before the limit
+   * is reached.
+   */
+  private long toLimit() {
+    return maxBytes - consumed();
+  }
+
+  @Override
+  public int available() throws IOException {
+    return (int) Math.min(toLimit(), countingIn.available());
+  }
+
+  @Override
+  public int read() throws IOException {
+    if (toLimit() > 0) {
+      return super.read();
+    } else {
+      return -1; // EOF.
+    }
+  }
+
+  @Override
+  public int read(byte [] buf) throws IOException {
+    return read(buf, 0, buf.length);
+  }
+
+  @Override
+  public int read(byte [] buf, int start, int count) throws IOException {
+    long limit = toLimit();
+    if (limit == 0) {
+      return -1; // EOF.
+    } else {
+      return super.read(buf, start, (int) Math.min(count, limit));
+    }
+  }
+}
+
diff --git a/src/java/org/apache/hadoop/sqoop/io/HdfsSplitOutputStream.java b/src/java/org/apache/hadoop/sqoop/io/HdfsSplitOutputStream.java
deleted file mode 100644
index 2ffbbb3b..00000000
--- a/src/java/org/apache/hadoop/sqoop/io/HdfsSplitOutputStream.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/**
- * Licensed to Cloudera, Inc. under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  Cloudera, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.sqoop.io;
-
-import java.io.OutputStream;
-import java.io.IOException;
-import java.util.zip.GZIPOutputStream;
-import java.util.Formatter;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-/**
- * An output stream that writes to HDFS, opening a new file after
- * a specified number of bytes have been written to the current one.
- */
-public class HdfsSplitOutputStream extends OutputStream {
-
-  public static final Log LOG = LogFactory.getLog(HdfsSplitOutputStream.class.getName());
-
-  private OutputStream writeStream;
-  private CountingOutputStream countingFilterStream;
-  private Configuration conf;
-  private Path destDir;
-  private String filePrefix;
-  private long cutoffBytes;
-  private boolean doGzip;
-  private int fileNum;
-
-  /**
-   * Create a new HdfsSplitOutputStream.
-   * @param conf the Configuration to use to interface with HDFS
-   * @param destDir the directory where the files will go (should already exist).
-   * @param filePrefix the first part of the filename, which will be appended by a number.
-   *    This file will be placed inside destDir.
-   * @param cutoff the approximate number of bytes to use per file
-   * @param doGzip if true, then output files will be gzipped and have a .gz suffix.
-   */
-  public HdfsSplitOutputStream(final Configuration conf, final Path destDir,
-      final String filePrefix, final long cutoff, final boolean doGzip) throws IOException {
-
-    this.conf = conf;
-    this.destDir = destDir;
-    this.filePrefix = filePrefix;
-    this.cutoffBytes = cutoff;
-    if (this.cutoffBytes < 0) {
-      this.cutoffBytes = 0; // splitting disabled.
-    }
-    this.doGzip = doGzip;
-    this.fileNum = 0;
-
-    openNextFile();
-  }
-
-  /** Initialize the OutputStream to the next file to write to.
-   */
-  private void openNextFile() throws IOException {
-    FileSystem fs = FileSystem.get(conf);
-
-    StringBuffer sb = new StringBuffer();
-    Formatter fmt = new Formatter(sb);
-    fmt.format("%05d", this.fileNum++);
-    String filename = filePrefix + fmt.toString();
-    if (this.doGzip) {
-      filename = filename + ".gz";
-    }
-    Path destFile = new Path(destDir, filename);
-    LOG.debug("Opening next output file: " + destFile);
-    if (fs.exists(destFile)) {
-       Path canonicalDest = destFile.makeQualified(fs);
-       throw new IOException("Destination file " + canonicalDest + " already exists");
-    }
-
-    OutputStream fsOut = fs.create(destFile);
-
-    // Count how many actual bytes hit HDFS.
-    this.countingFilterStream = new CountingOutputStream(fsOut);
-
-    if (this.doGzip) {
-      // Wrap that in a Gzip stream.
-      this.writeStream = new GZIPOutputStream(this.countingFilterStream);
-    } else {
-      // Write to the counting stream directly.
-      this.writeStream = this.countingFilterStream;
-    }
-  }
-
-  /**
-   * @return true if allowSplit() would actually cause a split.
-   */
-  public boolean wouldSplit() {
-    return this.cutoffBytes > 0
-        && this.countingFilterStream.getBytesWritten() >= this.cutoffBytes;
-  }
-
-  /** If we've written more to the disk than the user's split size,
-   * open the next file.
-   */
-  private void checkForNextFile() throws IOException {
-    if (wouldSplit()) {
-      LOG.debug("Starting new split");
-      this.writeStream.flush();
-      this.writeStream.close();
-      openNextFile();
-    }
-  }
-
-  /** Defines a point in the stream when it is acceptable to split to a new file;
-      e.g., the end of a record.
-    */
-  public void allowSplit() throws IOException {
-    checkForNextFile();
-  }
-
-  public void close() throws IOException {
-    this.writeStream.close();
-  }
-
-  public void flush() throws IOException {
-    this.writeStream.flush();
-  }
-
-  public void write(byte [] b) throws IOException {
-    this.writeStream.write(b);
-  }
-
-  public void write(byte [] b, int off, int len) throws IOException {
-    this.writeStream.write(b, off, len);
-  }
-
-  public void write(int b) throws IOException {
-    this.writeStream.write(b);
-  }
-}
diff --git a/src/java/org/apache/hadoop/sqoop/io/LobFile.java b/src/java/org/apache/hadoop/sqoop/io/LobFile.java
new file mode 100644
index 00000000..dd163b12
--- /dev/null
+++ b/src/java/org/apache/hadoop/sqoop/io/LobFile.java
@@ -0,0 +1,1802 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop.io;
+
+import java.io.BufferedOutputStream;
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.rmi.server.UID;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.AbstractMap;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.commons.io.output.CloseShieldOutputStream;
+import org.apache.commons.io.output.CountingOutputStream;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.io.compress.CompressorStream;
+import org.apache.hadoop.io.compress.Decompressor;
+import org.apache.hadoop.io.compress.DecompressorStream;
+
+/**
+ * File format which stores large object records.
+ * The format allows large objects to be read through individual InputStreams
+ * to allow reading without full materialization of a single record.
+ * Each record is assigned an id and can be accessed by id efficiently by
+ * consulting an index at the end of the file.
+ *
+ * The LobFile format is specified at:
+ * http://wiki.github.com/cloudera/sqoop/sip-3
+ */
+public class LobFile {
+
+  public static final Log LOG = LogFactory.getLog(LobFile.class.getName());
+
+  public static final int LATEST_LOB_VERSION = 0;
+  static final char [] HEADER_ID_STR = { 'L', 'O', 'B' };
+
+  // Value for entryId to write to the beginning of an IndexSegment. 
+  static final long SEGMENT_HEADER_ID = -1;
+
+  // Value for entryId to write before the finale.
+  static final long SEGMENT_OFFSET_ID = -2;
+
+  // Value for entryID to write before the IndexTable
+  static final long INDEX_TABLE_ID = -3; 
+
+  /**
+   * Represents a header block in a LobFile. Can write a new header
+   * block (and generate a record start mark), or read an existing
+   * header block.
+   */
+  private static class LobFileHeader implements Writable {
+
+    private int version;
+    private RecordStartMark startMark;
+    private MetaBlock metaBlock;
+
+    /**
+     * Create a new LobFileHeader.
+     */
+    public LobFileHeader() {
+      this.version = LATEST_LOB_VERSION;
+      this.startMark = new RecordStartMark();
+      this.metaBlock = new MetaBlock();  
+    }
+
+    /**
+     * Read a LobFileHeader from an existing file.
+     */
+    public LobFileHeader(DataInput in) throws IOException {
+      readFields(in);
+    }
+
+    /**
+     * Write a LobFile header to an output sink.
+     */
+    public void write(DataOutput out) throws IOException {
+      // Start with the file type identification.
+      for (char c : HEADER_ID_STR) {
+        out.writeByte((int) c);
+      }
+
+      // Write the format version
+      WritableUtils.writeVInt(out, this.version);
+
+      startMark.write(out);
+      metaBlock.write(out);
+    }
+
+    public void readFields(DataInput in) throws IOException {
+      char [] chars = new char[3];
+      for (int i = 0; i < 3; i++) {
+        chars[i] = (char) in.readByte();
+      }
+
+      // Check that these match what we expect. Throws IOE if not.
+      checkHeaderChars(chars);
+
+      this.version = WritableUtils.readVInt(in);
+      if (this.version != LATEST_LOB_VERSION) {
+        // Right now we only have one version we can handle.
+        throw new IOException("Unexpected LobFile version " + this.version);
+      }
+
+      this.startMark = new RecordStartMark(in);
+      this.metaBlock = new MetaBlock(in);
+    }
+
+    /**
+     * Checks that a header array matches the standard LobFile header.
+     * Additional data at the end of the headerStamp is ignored.
+     * @param headerStamp the header bytes received from the file.
+     * @throws IOException if it doesn't.
+     */
+    private void checkHeaderChars(char [] headerStamp) throws IOException {
+      if (headerStamp.length != HEADER_ID_STR.length) {
+        throw new IOException("Invalid LobFile header stamp: expected length "
+            + HEADER_ID_STR.length);
+      }
+      for (int i = 0; i < HEADER_ID_STR.length; i++) {
+        if (headerStamp[i] != HEADER_ID_STR[i]) {
+          throw new IOException("Invalid LobFile header stamp");
+        }
+      }
+    }
+
+    /**
+     * @return the format version number for this LobFile
+     */
+    public int getVersion() {
+      return version;
+    }
+
+    /**
+     * @return the RecordStartMark for this LobFile.
+     */
+    public RecordStartMark getStartMark() {
+      return startMark;
+    }
+
+    /**
+     * @return the MetaBlock for this LobFile.
+     */
+    public MetaBlock getMetaBlock() {
+      return metaBlock;
+    }
+  }
+
+  /**
+   * Holds a RecordStartMark -- a 16 byte randomly-generated
+   * sync token. Can read a RSM from an input source, or can
+   * generate a new one.
+   */
+  private static class RecordStartMark implements Writable {
+
+    // This is a 16-byte array.
+    public static final int START_MARK_LENGTH = 16;
+
+    private byte [] startBytes;
+
+    public RecordStartMark() {
+      generateStartMark();
+    }
+
+    public RecordStartMark(DataInput in) throws IOException {
+      readFields(in);
+    }
+
+    public byte [] getBytes() {
+      byte [] out = new byte[START_MARK_LENGTH];
+      System.arraycopy(this.startBytes, 0, out, 0, START_MARK_LENGTH);
+      return out;
+    }
+
+    public void readFields(DataInput in) throws IOException {
+      this.startBytes = new byte[START_MARK_LENGTH];
+      in.readFully(this.startBytes);
+    }
+
+    public void write(DataOutput out) throws IOException {
+      out.write(this.startBytes);
+    }
+
+    /**
+     * Generate a new random RecordStartMark.
+     */
+    private void generateStartMark() {
+      try {
+        MessageDigest digester = MessageDigest.getInstance("MD5");
+        long time = System.currentTimeMillis();
+        digester.update((new UID() + "@" + time).getBytes());
+        this.startBytes = digester.digest();
+      } catch (NoSuchAlgorithmException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  /**
+   * Represents the metadata block stored in the header of a LobFile.
+   */
+  private static class MetaBlock extends AbstractMap<String, BytesWritable>
+      implements Writable {
+
+    // Strings which typically appear in the metablock have canonical names.
+    public static final String ENTRY_ENCODING_KEY = "EntryEncoding";
+    public static final String COMPRESSION_CODEC_KEY = "CompressionCodec";
+    public static final String ENTRIES_PER_SEGMENT_KEY = "EntriesPerSegment";
+
+    // Standard entry encodings.
+    public static final String CLOB_ENCODING = "CLOB";
+    public static final String BLOB_ENCODING = "BLOB";
+
+    private Map<String, BytesWritable> entries;
+
+    public MetaBlock() {
+      entries = new TreeMap<String, BytesWritable>();
+    }
+
+    public MetaBlock(DataInput in) throws IOException {
+      entries = new TreeMap<String, BytesWritable>();
+      readFields(in);
+    }
+
+    public MetaBlock(Map<String, BytesWritable> map) {
+      entries = new TreeMap<String, BytesWritable>();
+      for (Map.Entry<String, BytesWritable> entry : map.entrySet()) {
+        entries.put(entry.getKey(), entry.getValue());
+      }
+    }
+    
+    @Override
+    public Set<Map.Entry<String, BytesWritable>> entrySet() {
+      return entries.entrySet();
+    }
+
+    @Override
+    public BytesWritable put(String k, BytesWritable v) {
+      BytesWritable old = entries.get(k);
+      entries.put(k, v);
+      return old;
+    }
+
+    public BytesWritable put(String k, String v) {
+      try {
+        return put(k, new BytesWritable(v.getBytes("UTF-8")));
+      } catch (UnsupportedEncodingException uee) {
+        // Shouldn't happen; UTF-8 is always supported.
+        throw new RuntimeException(uee);
+      }
+    }
+
+    @Override
+    public BytesWritable get(Object k) {
+      return entries.get(k);
+    }
+
+    public String getString(Object k) {
+      BytesWritable bytes = get(k);
+      if (null == bytes) {
+        return null;
+      } else {
+        try {
+          return new String(bytes.getBytes(), 0, bytes.getLength(), "UTF-8");
+        } catch (UnsupportedEncodingException uee) {
+          // Shouldn't happen; UTF-8 is always supported.
+          throw new RuntimeException(uee);
+        }
+      }
+    }
+
+    public void readFields(DataInput in) throws IOException {
+      int numEntries = WritableUtils.readVInt(in);
+      entries.clear();
+      for (int i = 0; i < numEntries; i++) {
+        String key = Text.readString(in);
+        BytesWritable val = new BytesWritable();
+        val.readFields(in);
+        entries.put(key, val);
+      }
+    }
+
+    public void write(DataOutput out) throws IOException {
+      int numEntries = entries.size();
+      WritableUtils.writeVInt(out, numEntries);
+      for (Map.Entry<String, BytesWritable> entry : entries.entrySet()) {
+        Text.writeString(out, entry.getKey());
+        entry.getValue().write(out);
+      }
+    }
+  }
+
+  /**
+   * Class that represents the IndexSegment entries in a LobIndex.
+   */
+  private static class IndexSegment implements Writable {
+
+    // The main body of the IndexSegment: the record lengths
+    // of all the records in the IndexSegment.
+    private BytesWritable recordLenBytes;
+
+    // The length of the previously recorded field (used when
+    // generating an index). Intermediate state used in calculation
+    // of the lastIndexOffset.
+    private long prevLength;
+
+    // Used to write VLong-encoded lengths into a temp
+    // array, which are then copied into recordLenBytes.
+    private DataOutputBuffer outputBuffer;
+
+    // The IndexTableEntry that describes this IndexSegment in the IndexTable.
+    private IndexTableEntry tableEntry;
+
+    public IndexSegment(IndexTableEntry tableEntry) {
+      this.recordLenBytes = new BytesWritable();
+      this.outputBuffer = new DataOutputBuffer(10); // max VLong size.
+      this.tableEntry = tableEntry;
+    }
+
+    /**
+     * Read an IndexSegment from an existing file.
+     */
+    public IndexSegment(IndexTableEntry tableEntry, DataInput in)
+        throws IOException {
+      this.recordLenBytes = new BytesWritable();
+      this.outputBuffer = new DataOutputBuffer(10);
+      this.tableEntry = tableEntry;
+      readFields(in);
+    }
+
+    /**
+     * @return the IndexTableEntry describing this IndexSegment in the
+     * IndexTable.
+     */
+    public IndexTableEntry getTableEntry() {
+      return tableEntry;
+    }
+
+    /**
+     * Add a recordLength to the recordLenBytes array.
+     */
+    public void addRecordLen(long recordLen) throws IOException {
+      // Allocate space for the new bytes.
+      int numBytes = WritableUtils.getVIntSize(recordLen);
+      recordLenBytes.setSize(recordLenBytes.getLength() + numBytes);
+
+      // Write the new bytes into a temporary buffer wrapped in a DataOutput.
+      outputBuffer.reset();
+      WritableUtils.writeVLong(outputBuffer, recordLen);
+
+      // Then copy those new bytes into the end of the recordLenBytes array.
+      System.arraycopy(outputBuffer.getData(), 0, recordLenBytes.getBytes(),
+          recordLenBytes.getLength() - numBytes, numBytes);
+
+      // Now that we've added a new recordLength to the array,
+      // it's the last index. We need to calculate its offset.
+      // This is based on how long the previous record was.
+      this.tableEntry.setLastIndexOffset(
+          this.tableEntry.getLastIndexOffset() + this.prevLength);
+
+      // Save this record's length (unserialized) for calculating
+      // lastIndexOffset for the next record.
+      this.prevLength = recordLen;
+    }
+
+    public void write(DataOutput out) throws IOException {
+      // Write the SEGMENT_HEADER_ID to distinguish this from a LobRecord.
+      WritableUtils.writeVLong(out, SEGMENT_HEADER_ID);
+      
+      // The length of the main body of the segment is the length of the
+      // data byte array.
+      int segmentBytesLen = recordLenBytes.getLength();
+      WritableUtils.writeVLong(out, segmentBytesLen);
+
+      // Write the body of the segment.
+      out.write(recordLenBytes.getBytes(), 0, segmentBytesLen);
+    }
+
+    public void readFields(DataInput in) throws IOException {
+      // After the RecordStartMark, we expect to get a SEGMENT_HEADER_ID (-1).
+      long segmentId = WritableUtils.readVLong(in);
+      if (SEGMENT_HEADER_ID != segmentId) {
+        throw new IOException("Expected segment header id " + SEGMENT_HEADER_ID
+            + "; got " + segmentId);
+      }
+
+      // Get the length of the rest of the segment, in bytes.
+      long length = WritableUtils.readVLong(in);
+
+      // Now read the actual main byte array.
+      if (length > Integer.MAX_VALUE) {
+        throw new IOException("Unexpected oversize data array length: "
+            + length);
+      } else if (length < 0) {
+        throw new IOException("Unexpected undersize data array length: "
+            + length);
+      }
+      byte [] segmentData = new byte[(int) length];
+      in.readFully(segmentData);
+      recordLenBytes = new BytesWritable(segmentData);
+
+      reset(); // Reset the iterator allowing the user to yield offset/lengths.
+    }
+    
+
+    // The following methods are used by a Reader to walk through the index
+    // segment and get data about the records described in this segment of
+    // the index.
+    
+    private DataInputBuffer dataInputBuf;
+
+    // The following two fields are advanced by the next() method.
+    private long curOffset; // offset into the file of the current record.
+    private long curLen; // length of the current record in bytes.
+
+    // Used to allow rewindOnce() to go backwards a single position in the
+    // iterator.
+    private int prevInputBufPos; // prev offset into dataInputBuf.
+    private long prevOffset;
+    private long prevLen;
+
+    /**
+     * Resets the record index iterator.
+     */
+    public void reset() {
+      this.dataInputBuf = null;
+    }
+
+    /**
+     * Aligns the iteration capability to return info about the next
+     * record in the IndexSegment. Must be called before the first
+     * record.
+     * @return true if there is another record described in this IndexSegment.
+     */
+    public boolean next() {
+      this.prevOffset = this.curOffset;
+      if (null == dataInputBuf) {
+        // We need to set up the iterator; this is the first use.
+        if (null == recordLenBytes) {
+          return false; // We don't have any records?
+        }
+
+        this.dataInputBuf = new DataInputBuffer();
+        this.dataInputBuf.reset(recordLenBytes.getBytes(),
+            0, recordLenBytes.getLength());
+
+        this.curOffset = this.tableEntry.getFirstIndexOffset();
+        this.prevOffset = 0;
+      } else {
+        this.curOffset += this.curLen;
+      }
+
+      boolean available = dataInputBuf.getPosition() < dataInputBuf.getLength();
+      if (available) {
+        this.prevInputBufPos = dataInputBuf.getPosition();
+        // Then read out the next record length.
+        try {
+          this.prevLen = this.curLen;
+          this.curLen = WritableUtils.readVLong(dataInputBuf);
+        } catch (IOException ioe) {
+          // Shouldn't happen; data in DataInputBuffer is materialized.
+          throw new RuntimeException(ioe);
+        }
+      }
+
+      return available;
+    }
+
+    /**
+     * Undoes a single call to next(). This cannot be called twice in a row;
+     * before calling this again, next() must be called in the interim. This
+     * makes a subsequent call to next() yield the same iterated values as the
+     * previous call.
+     */
+    public void rewindOnce() {
+      // Move the buffer backwards so we deserialize the same VLong with
+      // the next call.
+      if (prevInputBufPos == 0) {
+        // We actually rewound the first next() in the iterator.
+        // Just reset the iterator to the beginning. Otherwise we'll
+        // backfill it with bogus data.
+        reset();
+      } else {
+        // Use the normal codepath; move the serialization buffer
+        // backwards and restores the previously yielded values.
+        dataInputBuf.reset(recordLenBytes.getBytes(), prevInputBufPos,
+            recordLenBytes.getLength() - prevInputBufPos);
+
+        // And restore the previously-yielded values.
+        this.curLen = this.prevLen;
+        this.curOffset = this.prevOffset;
+      }
+    }
+
+    /**
+     * Returns the length of the current record.
+     * You must call next() and it must return true before calling this method.
+     * @return the length in bytes of the current record.
+     */
+    public long getCurRecordLen() {
+      return curLen;
+    }
+
+    /**
+     * Returns the offset of the current record from the beginning of the file.
+     * You must call next() and it must return true before calling this method.
+     * @return the offset in bytes from the beginning of the file for the
+     * current record.
+     */
+    public long getCurRecordStart() {
+      return curOffset;
+    }
+  }
+
+  /**
+   * Describes an IndexSegment. This is one entry in the IndexTable. It
+   * holds the physical location of the IndexSegment in the file, as well
+   * as the range of entryIds and byte ranges corresponding to records
+   * described by the index subset in the IndexSegment. 
+   */
+  private static class IndexTableEntry implements Writable {
+    private long segmentOffset;
+    private long firstIndexId;
+    private long firstIndexOffset;
+    private long lastIndexOffset;
+
+    public IndexTableEntry() {
+    }
+
+    public IndexTableEntry(DataInput in) throws IOException {
+      readFields(in);
+    }
+
+    private void setSegmentOffset(long offset) {
+      this.segmentOffset = offset;
+    }
+
+    private void setFirstIndexId(long id) {
+      this.firstIndexId = id;
+    }
+
+    private void setFirstIndexOffset(long offset) {
+      this.firstIndexOffset = offset;
+    }
+
+    private void setLastIndexOffset(long offset) {
+      this.lastIndexOffset = offset;
+    }
+
+    public void write(DataOutput out) throws IOException {
+      WritableUtils.writeVLong(out, segmentOffset);
+      WritableUtils.writeVLong(out, firstIndexId);
+      WritableUtils.writeVLong(out, firstIndexOffset);
+      WritableUtils.writeVLong(out, lastIndexOffset);
+    }
+
+    public void readFields(DataInput in) throws IOException {
+      segmentOffset = WritableUtils.readVLong(in);
+      firstIndexId = WritableUtils.readVLong(in);
+      firstIndexOffset = WritableUtils.readVLong(in);
+      lastIndexOffset = WritableUtils.readVLong(in);
+    }
+
+    /**
+     * @return the entryId of the first record indexed by this segment.
+     */
+    public long getFirstIndexId() {
+      return this.firstIndexId;
+    }
+
+    /**
+     * @return the offset of the first record indexed by this segment.
+     */
+    public long getFirstIndexOffset() {
+      return this.firstIndexOffset;
+    }
+
+    /**
+     * @return the offset of the last record indexed by this segment.
+     */
+    public long getLastIndexOffset() {
+      return this.lastIndexOffset;
+    }
+
+    /**
+     * @return the offset from the start of the file of the IndexSegment
+     * data itself.
+     */
+    public long getSegmentOffset() {
+      return this.segmentOffset;
+    }
+
+    /**
+     * Inform whether the user's requested offset corresponds
+     * to a record that starts in this IndexSegment. If this
+     * returns true, the requested offset may actually be in
+     * a previous IndexSegment.
+     * @param off the offset of the start of a record to test.
+     * @return true if the user's requested offset is in this
+     * or a previous IndexSegment.
+     */
+    public boolean containsOffset(long off) {
+      return off <= getLastIndexOffset();
+    }
+  }
+
+  /**
+   * Stores the locations and ranges indexed by each IndexSegment.
+   */
+  private static class IndexTable
+      implements Iterable<IndexTableEntry>, Writable {
+    private List<IndexTableEntry> tableEntries;
+
+    public IndexTable() {
+      tableEntries = new ArrayList<IndexTableEntry>();
+    }
+
+    public IndexTable(DataInput in) throws IOException {
+      readFields(in);
+    }
+
+    public void readFields(DataInput in) throws IOException {
+      long recordTypeId = WritableUtils.readVLong(in);
+      if (recordTypeId != INDEX_TABLE_ID) {
+        // We expected to read an IndexTable.
+        throw new IOException("Expected IndexTable; got record with typeId="
+            + recordTypeId);
+      }
+
+      int tableCount = WritableUtils.readVInt(in);
+
+      tableEntries = new ArrayList<IndexTableEntry>(tableCount);
+      for (int i = 0; i < tableCount; i++) {
+        tableEntries.add(new IndexTableEntry(in));
+      }
+    }
+
+    public void write(DataOutput out) throws IOException {
+      // Start with the record type id.
+      WritableUtils.writeVLong(out, INDEX_TABLE_ID);
+
+      // Then the count of the records.
+      WritableUtils.writeVInt(out, tableEntries.size());
+
+      // Followed by the table itself.
+      for (IndexTableEntry entry : tableEntries) {
+        entry.write(out);
+      }
+    }
+
+    public void add(IndexTableEntry entry) {
+      tableEntries.add(entry);
+    }
+
+    public IndexTableEntry get(int i) {
+      return tableEntries.get(i);
+    }
+
+    public int size() {
+      return tableEntries.size();
+    }
+
+    public Iterator<IndexTableEntry> iterator() {
+      return tableEntries.iterator();
+    }
+  }
+
+  /**
+   * Class that writes out a LobFile. Instantiate via LobFile.create().
+   */
+  public static abstract class Writer implements Closeable {
+
+    /**
+     * If this Writer is writing to a physical LobFile, then this returns
+     * the file path it is writing to. Otherwise it returns null.
+     * @return the fully-qualified path being written to by this writer.
+     */
+    public abstract Path getPath();
+
+    /**
+     * Finishes writing the LobFile and closes underlying handles.
+     */
+    public abstract void close() throws IOException;
+
+    @Override
+    protected synchronized void finalize() throws Throwable {
+      close();
+    }
+
+    /**
+     * Terminates the current record and writes any trailing zero-padding
+     * required by the specified record size.
+     * This is implicitly called between consecutive writeBlobRecord() /
+     * writeClobRecord() calls.
+     */
+    public abstract void finishRecord() throws IOException;
+
+    /**
+     * Declares a new BLOB record to be written to the file.
+     * @param len the "claimed" number of bytes that will be written to
+     * this record. The actual number of bytes may differ.
+     */
+    public abstract OutputStream writeBlobRecord(long len) throws IOException;
+
+    /**
+     * Declares a new CLOB record to be written to the file.
+     * @param len the claimed number of characters that will be written to
+     * this record. The actual number of characters may differ.
+     */
+    public abstract java.io.Writer writeClobRecord(long len)
+        throws IOException;
+
+    /**
+     * Report the current position in the output file
+     * @return the number of bytes written through this Writer.
+     */
+    public abstract long tell() throws IOException;
+
+    /**
+     * Checks whether an underlying stream is present or null.
+     * @param out the stream to check for null-ness.
+     * @throws IOException if out is null.
+     */
+    protected void checkForNull(OutputStream out) throws IOException {
+      if (null == out) {
+        throw new IOException("Writer has been closed.");
+      }
+    }
+  }
+
+  /**
+   * Concrete writer implementation for LobFile format version 0.
+   * Instantiate via LobFile.create().
+   */
+  private static class V0Writer extends Writer {
+    public static final Log LOG = LogFactory.getLog(
+        V0Writer.class.getName());
+
+    private Configuration conf;
+    private Path path;
+    private boolean isCharData;
+    private LobFileHeader header;
+
+    private String codecName;
+    private CompressionCodec codec;
+    private Compressor compressor;
+
+    // The LobIndex we are constructing.
+    private LinkedList<IndexSegment> indexSegments;
+    private int entriesInSegment; // number of entries in the current IndexSegment.
+    private IndexTable indexTable;
+
+    // Number of entries that can be written to a single IndexSegment.
+    private int maxEntriesPerSegment; 
+
+    // By default we write this many entries per IndexSegment.
+    static final int DEFAULT_MAX_SEGMENT_ENTRIES = 4096;
+    
+    // Our OutputStream to the underlying file. 
+    private DataOutputStream out;
+
+    // 'out' is layered on top of this stream, which gives us a count
+    // of how much data we've written so far.
+    private CountingOutputStream countingOut;
+
+    // State regarding the current record being written.
+    private long curEntryId; // entryId of the current LOB being written.
+    private long curClaimedLen; // The user claims a length for a record.
+
+    // The user's OutputStream and/or Writer that writes to us.
+    private OutputStream userOutputStream;
+    private java.io.Writer userWriter;
+
+    // The userCountingOutputStream may be the same as userOutputStream;
+    // but if the user is writing through a compressor, it is actually
+    // underneath of it. This tells us how many compressed bytes were
+    // really written.
+    private CountingOutputStream userCountingOutputStream;
+
+    /**
+     * Creates a LobFile Writer for file format version 0.
+     * @param p the path to create.
+     * @param conf the configuration to use to interact with the filesystem.
+     * @param isCharData true if this is for CLOBs, false for BLOBs.
+     * @param codecName the compression codec to use (or null for none).
+     * @param entriesPerSegment the number of index entries per IndexSegment.
+     */
+    V0Writer(Path p, Configuration conf, boolean isCharData,
+        String codecName, int entriesPerSegment) throws IOException {
+
+      this.path = LobReaderCache.qualify(p, conf);
+      this.conf = conf;
+      this.isCharData = isCharData;
+      this.header = new LobFileHeader();
+      this.indexSegments = new LinkedList<IndexSegment>();
+      this.indexTable = new IndexTable();
+      this.maxEntriesPerSegment = entriesPerSegment;
+
+      this.codecName = codecName;
+      if (this.codecName != null) {
+        this.codec = CodecMap.getCodec(codecName, conf);
+        if (null != this.codec) {
+          this.compressor = codec.createCompressor();
+        }
+      }
+      
+      init();
+    }
+
+    /**
+     * Open the file and write its header.
+     */
+    private void init() throws IOException {
+      FileSystem fs = this.path.getFileSystem(conf);
+      FSDataOutputStream fsOut = fs.create(this.path);
+      this.countingOut = new CountingOutputStream(
+          new BufferedOutputStream(fsOut));
+      this.out = new DataOutputStream(this.countingOut);
+
+      // put any necessary config strings into the header.
+      MetaBlock m = this.header.getMetaBlock();
+      if (isCharData) {
+        m.put(MetaBlock.ENTRY_ENCODING_KEY, MetaBlock.CLOB_ENCODING);
+      } else {
+        m.put(MetaBlock.ENTRY_ENCODING_KEY, MetaBlock.BLOB_ENCODING);
+      }
+
+      if (null != codec) {
+        m.put(MetaBlock.COMPRESSION_CODEC_KEY, this.codecName);
+      }
+
+      // Serialize the value of maxEntriesPerSegment as a VInt in a byte array
+      // and put that into the metablock as ENTRIES_PER_SEGMENT_KEY.
+      int segmentBufLen = WritableUtils.getVIntSize(this.maxEntriesPerSegment);
+      DataOutputBuffer entriesPerSegBuf = new DataOutputBuffer(segmentBufLen);
+      WritableUtils.writeVInt(entriesPerSegBuf, this.maxEntriesPerSegment);
+      byte [] entriesPerSegArray =
+          Arrays.copyOf(entriesPerSegBuf.getData(), segmentBufLen);
+      m.put(MetaBlock.ENTRIES_PER_SEGMENT_KEY,
+          new BytesWritable(entriesPerSegArray));
+
+      // Write the file header to the file.
+      this.header.write(out);
+
+      // Now we're ready to accept record data from the user.
+    }
+
+    @Override
+    /** {@inheritDoc} */
+    public Path getPath() {
+      return this.path;
+    }
+
+    @Override
+    /**
+     * {@inheritDoc}
+     */
+    public long tell() throws IOException {
+      checkForNull(this.out);
+      this.out.flush();
+      return this.countingOut.getByteCount();
+    }
+
+    @Override
+    /**
+     * {@inheritDoc}
+     */
+    public void close() throws IOException {
+      finishRecord();
+      writeIndex();
+      if (this.out != null) {
+        this.out.close();
+        this.out = null;
+      }
+
+      if (this.countingOut != null) {
+        this.countingOut.close();
+        this.countingOut = null;
+      }
+    }
+
+    @Override
+    /**
+     * {@inheritDoc}
+     */
+    public void finishRecord() throws IOException {
+      if (null != this.userWriter) {
+        this.userWriter.close();
+        this.userWriter = null;
+      }
+
+      if (null != this.userCountingOutputStream) {
+
+        // If there is a wrapping stream for compression,
+        // close this first.
+        if (null != this.userOutputStream
+            && this.userOutputStream != this.userCountingOutputStream) {
+          this.userOutputStream.close();
+        }
+
+        // Now close the "main" stream.
+        this.userCountingOutputStream.close();
+
+        // Write the true length of the current record to the index.
+        updateIndex(this.userCountingOutputStream.getByteCount()
+            + RecordStartMark.START_MARK_LENGTH
+            + WritableUtils.getVIntSize(curEntryId)
+            + WritableUtils.getVIntSize(curClaimedLen));
+
+        this.userOutputStream = null;
+        this.userCountingOutputStream = null;
+      }
+
+      if (null != this.out) {
+        out.flush();
+      }
+    }
+
+    /**
+     * Write in the current IndexSegment, the true compressed length of the
+     * record we just finished writing.
+     * @param curRecordLen the true length in bytes of the compressed record.
+     */
+    private void updateIndex(long curRecordLen) throws IOException {
+      LOG.debug("Adding index entry: id=" + curEntryId
+          + "; len=" + curRecordLen);
+      indexSegments.getLast().addRecordLen(curRecordLen);
+      entriesInSegment++;
+      curEntryId++;
+    }
+
+    /**
+     * Write the index itself to the file.
+     */
+    private void writeIndex() throws IOException {
+
+      // Write out all the segments in turn.
+      // As we do so, reify their offsets into the IndexTable.
+      for (IndexSegment segment : indexSegments) {
+        long segmentOffset = tell();
+        segment.getTableEntry().setSegmentOffset(segmentOffset);
+
+        header.getStartMark().write(out);
+        segment.write(out);
+      }
+
+      long indexTableStartPos = tell(); // Save for the end of the file.
+      LOG.debug("IndexTable offset: " + indexTableStartPos);
+
+      header.getStartMark().write(out);
+      indexTable.write(out); // write the IndexTable record.
+
+      // Write the finale that tells us where the IndexTable begins.
+      header.getStartMark().write(out);
+      WritableUtils.writeVLong(out, SEGMENT_OFFSET_ID);
+      WritableUtils.writeVLong(out, indexTableStartPos);
+    }
+
+    /**
+     * Prepare to index a new record that will soon be written to the file.
+     * If this is is the first record in the current IndexSegment, we need
+     * to record its entryId and the current file position.
+     */
+    private void startRecordIndex() throws IOException {
+      if (entriesInSegment == maxEntriesPerSegment
+          || indexSegments.size() == 0) {
+        // The current segment is full. Start a new one.
+        this.entriesInSegment = 0;
+        IndexTableEntry tableEntry = new IndexTableEntry();
+        IndexSegment curSegment = new IndexSegment(tableEntry);
+        this.indexSegments.add(curSegment);
+
+        long filePos = tell();
+        LOG.debug("Starting IndexSegment; first id=" + curEntryId
+            + "; off=" + filePos);
+        tableEntry.setFirstIndexId(curEntryId);
+        tableEntry.setFirstIndexOffset(filePos);
+        tableEntry.setLastIndexOffset(filePos);
+        this.indexTable.add(tableEntry);
+      }
+    }
+
+    @Override
+    /**
+     * {@inheritDoc}
+     */
+    public OutputStream writeBlobRecord(long claimedLen) throws IOException {
+      finishRecord(); // finish any previous record.
+      checkForNull(this.out);
+      startRecordIndex();
+      this.header.getStartMark().write(out);
+      LOG.debug("Starting new record; id=" + curEntryId
+          + "; claimedLen=" + claimedLen);
+      WritableUtils.writeVLong(out, curEntryId);
+      WritableUtils.writeVLong(out, claimedLen);
+      this.curClaimedLen = claimedLen;
+      this.userCountingOutputStream = new CountingOutputStream(
+          new CloseShieldOutputStream(out));
+      if (null == this.codec) {
+        // No codec; pass thru the same OutputStream to the user.
+        this.userOutputStream = this.userCountingOutputStream;
+      } else {
+        // Wrap our CountingOutputStream in a compressing OutputStream to
+        // give to the user.
+        this.compressor.reset();
+        this.userOutputStream = new CompressorStream(
+            this.userCountingOutputStream, compressor);
+      }
+
+      return this.userOutputStream;
+    }
+
+    @Override
+    /**
+     * {@inheritDoc}
+     */
+    public java.io.Writer writeClobRecord(long len) throws IOException {
+      if (!isCharData) {
+        throw new IOException(
+            "Can only write CLOB data to a Clob-specific LobFile");
+      }
+
+      // Get a binary handle to the record and wrap it in a java.io.Writer.
+      writeBlobRecord(len);
+      this.userWriter = new OutputStreamWriter(userOutputStream);
+      return this.userWriter;
+    }
+  }
+
+  /**
+   * Class that can read a LobFile. Create with LobFile.open().
+   */
+  public static abstract class Reader implements Closeable {
+    /**
+     * If this Reader is reading from a physical LobFile, then this returns
+     * the file path it is reading from. Otherwise it returns null.
+     * @return the fully-qualified path being read by this reader.
+     */
+    public abstract Path getPath();
+
+    /**
+     * Report the current position in the file
+     * @return the current offset from the start of the file in bytes.
+     */
+    public abstract long tell() throws IOException;
+
+    /**
+     * Move the file pointer to the first available full record beginning at
+     * position 'pos', relative to the start of the file.  After calling
+     * seek(), you will need to call next() to move to the record itself.
+     * @param pos the position to seek to or past.
+     */
+    public abstract void seek(long pos) throws IOException;
+
+    /**
+     * Advances to the next record in the file.
+     * @return true if another record exists, or false if the
+     * end of the file has been reached.
+     */
+    public abstract boolean next() throws IOException;
+
+    /**
+     * @return true if we have aligned the Reader (through a call to next())
+     * onto a record.
+     */
+    public abstract boolean isRecordAvailable();
+
+    /**
+     * Reports the length of the record to the user.
+     * If next() has not been called, or seek() has been called without
+     * a subsequent call to next(), or next() returned false, the return
+     * value of this method is undefined.
+     * @return the 'claimedLen' field of the current record. For
+     * character-based records, this is often in characters, not bytes.
+     * Records may have more bytes associated with them than are reported
+     * by this method, but never fewer.
+     */
+    public abstract long getRecordLen();
+
+    /**
+     * Return the entryId of the current record to the user.
+     * If next() has not been called, or seek() has been called without
+     * a subsequent call to next(), or next() returned false, the return
+     * value of this method is undefined.
+     * @return the 'entryId' field of the current record.
+     */
+    public abstract long getRecordId();
+
+    /**
+     * Return the byte offset at which the current record starts.
+     * If next() has not been called, or seek() has been called without
+     * a subsequent call to next(), or next() returned false, the return
+     * value of this method is undefined.
+     * @return the byte offset of the beginning of the current record.
+     */
+    public abstract long getRecordOffset();
+
+    /**
+     * @return an InputStream allowing the user to read the next binary
+     * record from the file.
+     */
+    public abstract InputStream readBlobRecord() throws IOException;
+
+    /**
+     * @return a java.io.Reader allowing the user to read the next character
+     * record from the file.
+     */
+    public abstract java.io.Reader readClobRecord() throws IOException;
+
+    /**
+     * Closes the reader.
+     */
+    public abstract void close() throws IOException;
+
+    /**
+     * Checks whether an underlying stream is present or null.
+     * @param in the stream to check for null-ness.
+     * @throws IOException if in is null.
+     */
+    protected void checkForNull(InputStream in) throws IOException {
+      if (null == in) {
+        throw new IOException("Reader has been closed.");
+      }
+    }
+
+    /**
+     * @return true if the Reader.close() method has been called.
+     */
+    public abstract boolean isClosed();
+
+    @Override
+    protected synchronized void finalize() throws Throwable {
+      close();
+    }
+  }
+
+  /**
+   * Reader implementation for LobFile format version 0. Acquire with
+   * LobFile.open().
+   */
+  private static class V0Reader extends Reader {
+    public static final Log LOG = LogFactory.getLog(
+        V0Reader.class.getName());
+
+    // Forward seeks of up to this size are performed by reading, not seeking.
+    private static final long MAX_CONSUMPTION_WIDTH = 512 * 1024;
+
+    private LobFileHeader header;
+
+    private Configuration conf;
+
+    // Codec to use to decompress the file.
+    private CompressionCodec codec;
+    private Decompressor decompressor;
+
+    // Length of the entire file.
+    private long fileLen;
+
+    // State bit set to true after we've called next() and successfully
+    // aligned on a record. If true, we can hand an InputStream back to
+    // the user.
+    private boolean isAligned;
+
+    // After we've aligned on a record, this contains the record's
+    // reported length. In the presence of compression, etc, this may
+    // not represent its true length in the file.
+    private long claimedRecordLen;
+
+    // After we've aligned on a record, this contains its entryId.
+    private long curEntryId; 
+
+    // After we've aligned on a record, this contains the offset of the
+    // beginning of its RSM from the start of the file.
+    private long curRecordOffset;
+
+    // After we've aligned on a record, this contains the record's
+    // true length from the index.
+    private long indexRecordLen;
+
+    // tmp buffer used to consume RecordStartMarks during alignment.
+    private byte [] tmpRsmBuf;
+
+    // The actual file stream itself, which we can move around (e.g. with
+    // seeking).
+    private FSDataInputStream underlyingInput;
+
+    // The data deserializer we typically place on top of this.
+    // If we use underlyingInput.seek(), then we instantiate a new
+    // dataIn on top of it.
+    private DataInputStream dataIn;
+
+    // The user accesses the current record through a stream memoized here.
+    // We retain a pointer here so that we can forcibly close the old
+    // userInputStream when they want to align on the next record.
+    private InputStream userInputStream;
+
+    // The current index segment to read record lengths from.
+    private IndexSegment curIndexSegment;
+
+    // The offset into the indexTable of the curIndexSegment.
+    private int curIndexSegmentId;
+
+    // The IndexTable that provides fast pointers to the IndexSegments.
+    private IndexTable indexTable;
+
+    // The path being opened.
+    private Path path;
+
+    // Users should use LobFile.open() instead of directly calling this.
+    V0Reader(Path path, Configuration conf, LobFileHeader header,
+        DataInputStream dis, FSDataInputStream stream, long fileLen)
+        throws IOException {
+      this.path = LobReaderCache.qualify(path, conf);
+      this.conf = conf;
+      this.header = header;
+      this.dataIn = dis;
+      this.underlyingInput = stream;
+      this.isAligned = false;
+      this.tmpRsmBuf = new byte[RecordStartMark.START_MARK_LENGTH];
+      this.fileLen = fileLen;
+      LOG.debug("Opening LobFile path: " + path);
+      openCodec();
+      openIndex();
+    }
+
+    /**
+     * If the user has specified a compression codec in the header metadata,
+     * create an instance of it.
+     */
+    private void openCodec() throws IOException {
+      String codecName = header.getMetaBlock().getString(
+          MetaBlock.COMPRESSION_CODEC_KEY);
+      if (null != codecName) {
+        LOG.debug("Decompressing file with codec: " + codecName);
+        this.codec = CodecMap.getCodec(codecName, conf);
+        if (null != this.codec) {
+          this.decompressor = codec.createDecompressor();
+        }
+      }
+    }
+
+    /**
+     * Get the first index segment out of the file; determine
+     * where that is by loading the index locator at the end of
+     * the file.
+     */
+    private void openIndex() throws IOException {
+      // Jump to the end of the file.
+      // At the end of the file is a RSM followed by two VLongs;
+      // the first of these is the value -2 (one byte) and the
+      // second of these is the offset of the beginning of the index (up to
+      // 9 bytes).
+      internalSeek(fileLen - RecordStartMark.START_MARK_LENGTH - 10);
+
+      byte [] finaleBuffer = new byte[RecordStartMark.START_MARK_LENGTH + 10];
+      this.dataIn.readFully(finaleBuffer);
+
+      // Figure out where in the finaleBuffer the RSM actually starts,
+      // as the finale might not fully fill the finaleBuffer.
+      int rsmStart = findRecordStartMark(finaleBuffer);
+      if (-1 == rsmStart) {
+        throw new IOException(
+            "Corrupt file index; could not find index start offset.");
+      }
+
+      // Wrap a buffer around those two vlongs.
+      int vlongStart = rsmStart + RecordStartMark.START_MARK_LENGTH;
+      DataInputBuffer inBuf = new DataInputBuffer();
+      inBuf.reset(finaleBuffer, vlongStart, finaleBuffer.length - vlongStart);
+
+      long offsetMarker = WritableUtils.readVLong(inBuf);
+      if (SEGMENT_OFFSET_ID != offsetMarker) {
+        // This isn't the correct signature; we got an RSM ahead of some
+        // other data.
+        throw new IOException("Invalid segment offset id: " + offsetMarker);
+      }
+
+      // This will contain the position of the IndexTable.
+      long indexTableStart = WritableUtils.readVLong(inBuf);
+      LOG.debug("IndexTable begins at " + indexTableStart);
+
+      readIndexTable(indexTableStart);
+
+      // Set up to read records from the beginning of the file. This
+      // starts with the first IndexSegment.
+      curIndexSegmentId = 0;
+      loadIndexSegment();
+
+      // This has moved the file pointer all over but we don't need to
+      // worry about resetting it now. The next() method will seek the
+      // file pointer to the first record when the user is ready to
+      // consume it.
+    }
+
+    /**
+     * Load the entire IndexTable into memory and decode it.
+     */
+    private void readIndexTable(long indexTableOffset) throws IOException {
+      internalSeek(indexTableOffset);
+
+      // Read the RecordStartMark ahead of the IndexTable.
+      this.dataIn.readFully(tmpRsmBuf);
+      if (!matchesRsm(tmpRsmBuf)) {
+        throw new IOException("Expected record start mark before IndexTable");
+      }
+
+      this.indexTable = new IndexTable(dataIn);
+    }
+
+    /**
+     * Ingest the next IndexSegment.
+     */
+    private void readNextIndexSegment() throws IOException {
+      this.curIndexSegmentId++;
+      loadIndexSegment();
+    }
+
+    /**
+     * Load curIndexSegment with the segment specified by curIndexSegmentId.
+     * The file pointer will be moved to the position after this segment.
+     * If the segment id does not exist, then the curIndexSegment will be
+     * set to null.
+     */
+    private void loadIndexSegment() throws IOException {
+      if (indexTable.size() <= curIndexSegmentId || curIndexSegmentId < 0) {
+        // We've iterated past the last IndexSegment. Set this to null
+        // and return; the next() method will then return false.
+        this.curIndexSegment = null;
+        return;
+      }
+
+      // Otherwise, seek to the segment and load it.
+      IndexTableEntry tableEntry = indexTable.get(curIndexSegmentId);
+      long segmentOffset = tableEntry.getSegmentOffset();
+      internalSeek(segmentOffset);
+      readPositionedIndexSegment();
+    }
+
+    /**
+     * When the underlying stream is aligned on the RecordStartMark
+     * ahead of an IndexSegment, read in the next IndexSegment.
+     * After this method the curIndexSegment contains the next
+     * IndexSegment to read in the file; if the entire index has been
+     * read in this fastion, curIndexSegment will be null.
+     */
+    private void readPositionedIndexSegment() throws IOException {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Reading index segment at " + tell());
+      }
+
+      // Read the RecordStartMark ahead of the IndexSegment.
+      this.dataIn.readFully(tmpRsmBuf);
+      if (!matchesRsm(tmpRsmBuf)) {
+        throw new IOException("Expected record start mark before IndexSegment");
+      }
+
+      // Read the IndexSegment proper.
+      this.curIndexSegment = new IndexSegment(
+          this.indexTable.get(curIndexSegmentId), this.dataIn);
+    }
+
+    /**
+     * @return true if the bytes in 'buf' starting at 'offset' match
+     * the RecordStartMark.
+     * @param rsm the RecordStartMark
+     * @param buf the buffer to check
+     * @param offset the offset into buf to begin checking.
+     */
+    private boolean matchesRsm(byte [] rsm, byte [] buf, int offset) {
+      for (int i = 0; i < RecordStartMark.START_MARK_LENGTH; i++) {
+        if (buf[i + offset] != rsm[i]) {
+          return false; // Mismatch at position i.
+        }
+      }
+
+      return true; // Matched the whole thing.
+    }
+
+    private boolean matchesRsm(byte [] buf, int offset) {
+      return matchesRsm(this.header.getStartMark().getBytes(),
+          buf, offset);
+    }
+
+    private boolean matchesRsm(byte [] buf) {
+      return matchesRsm(buf, 0);
+    }
+
+    /**
+     * @return the offset in 'buf' where a RecordStartMark begins, or -1
+     * if the RecordStartMark is not present in the buffer. 
+     */
+    private int findRecordStartMark(byte [] buf) {
+      byte [] rsm = this.header.getStartMark().getBytes();
+
+      for (int i = 0; i < buf.length; i++) {
+        if (matchesRsm(rsm, buf, i)) {
+          return i;
+        }
+      }
+
+      return -1; // couldn't find it.
+    }
+
+    /** {@inheritDoc} */
+    public Path getPath() {
+      return this.path;
+    }
+
+    /** {@inheritDoc} */
+    public long tell() throws IOException {
+      checkForNull(this.underlyingInput);
+      return this.underlyingInput.getPos();
+    }
+
+    /** {@inheritDoc} */
+    public void seek(long pos) throws IOException {
+      closeUserStream();
+      checkForNull(this.underlyingInput);
+      this.isAligned = false;
+      searchForRecord(pos);
+    }
+
+    /**
+     * Search the index for the first record starting on or after 'start'.
+     * @param start the offset in the file where we should start looking
+     * for a record.
+     */
+    private void searchForRecord(long start) throws IOException {
+      LOG.debug("Looking for the first record at/after offset " + start);
+
+      // Scan through the IndexTable until we find the IndexSegment
+      // that contains the offset. 
+      for (int i = 0; i < indexTable.size(); i++) {
+        IndexTableEntry tableEntry = indexTable.get(i);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Checking index table entry for range: "
+              + tableEntry.getFirstIndexOffset() + ", "
+              + tableEntry.getLastIndexOffset());
+        }
+
+        if (tableEntry.containsOffset(start)) {
+          // Seek to the IndexSegment associated with this tableEntry.
+          curIndexSegmentId = i;
+          loadIndexSegment();
+          
+          // Use this index segment. The record index iterator
+          // is at the beginning of the IndexSegment, since we just
+          // read it in.
+          LOG.debug("Found matching index segment.");
+          while (this.curIndexSegment.next()) {
+            long curStart = this.curIndexSegment.getCurRecordStart();
+            if (curStart >= start) {
+              LOG.debug("Found seek target record with offset " + curStart);
+              // This is the first record to meet this criterion.
+              // Rewind the index iterator by one so that the next()
+              // method will do the right thing. next() will also
+              // take care of actually seeking to the correct position
+              // in the file to read the record proper.
+              this.curIndexSegment.rewindOnce();
+              return;
+            }
+          }
+
+          // If it wasn't actually in this IndexSegment, then we've
+          // got a corrupt IndexTableEntry; the entry represented that
+          // the segment ran longer than it actually does.
+          throw new IOException("IndexTableEntry claims last offset of "
+              + tableEntry.getLastIndexOffset()
+              + " but IndexSegment ends early."
+              + " The IndexTable appears corrupt.");
+        }
+      }
+
+      // If we didn't return inside the loop, then we've searched the entire
+      // file and it's not there. Advance the IndexSegment iterator to 
+      // the end of the road so that next() returns false.
+      this.curIndexSegmentId = indexTable.size();
+      loadIndexSegment();
+    }
+
+    /**
+     * Read data from the stream and discard it.
+     * @param numBytes number of bytes to read and discard.
+     */
+    private void consumeBytes(int numBytes) throws IOException {
+      int remaining = numBytes;
+      while (remaining > 0) {
+        int received = dataIn.skipBytes(remaining);
+        if (received < 1) {
+          throw new IOException("Could not consume additional bytes");
+        }
+        remaining -= received;
+      }
+    }
+
+    /**
+     * Seek to position 'pos' (offset from start of file). If this
+     * is nearby, actually just consume data from the underlying
+     * stream rather than doing a real seek.
+     * @param targetPos the position to seek to, expressed as an offset
+     * from the start of the file.
+     */
+    private void internalSeek(long targetPos) throws IOException {
+      long curPos = this.underlyingInput.getPos();
+      LOG.debug("Internal seek: target=" + targetPos + "; cur=" + curPos);
+      long distance = targetPos - curPos;
+      if (targetPos == curPos) {
+        LOG.debug("(no motion required)");
+        return; // We're already there!
+      } else if (targetPos > curPos && distance < MAX_CONSUMPTION_WIDTH) {
+        // We're "close enough" that we should just read it.
+        LOG.debug("Advancing by " + distance + " bytes.");
+        consumeBytes((int) distance);
+      } else {
+        LOG.debug("Direct seek to target");
+        this.underlyingInput.seek(targetPos);
+        this.dataIn = new DataInputStream(this.underlyingInput);
+      }
+    }
+
+    /**
+     * Close any stream to an open record that was opened by a user.
+     */
+    private void closeUserStream() throws IOException {
+      if (this.userInputStream != null) {
+        this.userInputStream.close();
+        this.userInputStream = null;
+      }
+    }
+
+    /** {@inheritDoc} */
+    public boolean next() throws IOException {
+      LOG.debug("Checking for next record");
+      checkForNull(this.underlyingInput);
+      // If the user has opened a record stream, it is now void.
+      closeUserStream();
+      this.isAligned = false; // false until proven true.
+
+      // Get the position of the next record start.
+      // Check the index: is there another record?
+      if (null == curIndexSegment) {
+        LOG.debug("Index is finished; false");
+        return false; // No index remains. Ergo, no more records.
+      }
+      boolean moreInSegment = curIndexSegment.next();
+      if (!moreInSegment) {
+        // The current IndexSegment has been exhausted. Move to the next.
+        LOG.debug("Loading next index segment.");
+        readNextIndexSegment();
+        if (null == curIndexSegment) {
+          LOG.debug("Index is finished; false");
+          return false; // No index; no records.
+        }
+
+        // Try again with the next IndexSegment.
+        moreInSegment = curIndexSegment.next();
+      }
+
+      if (!moreInSegment) {
+        // Nothing left in the last IndexSegment.
+        LOG.debug("Last index segment is finished; false.");
+        this.curIndexSegment = null;
+        return false; 
+      }
+
+      // Determine where the next record starts.
+      this.indexRecordLen = this.curIndexSegment.getCurRecordLen();
+      this.curRecordOffset = this.curIndexSegment.getCurRecordStart();
+
+      LOG.debug("Next record starts at position: " + this.curRecordOffset
+          + "; indexedLen=" + this.indexRecordLen);
+
+      // Make sure we're at the target position.
+      internalSeek(this.curRecordOffset);
+
+      // We are now on top of the next record's RecordStartMark.
+      // Consume the RSM and the record header.
+      this.dataIn.readFully(this.tmpRsmBuf);
+      if (!matchesRsm(tmpRsmBuf)) {
+        // No rsm? No dice.
+        throw new IOException("Index contains bogus offset.");
+      }
+
+      this.curEntryId = WritableUtils.readVLong(this.dataIn);
+      if (this.curEntryId < 0) {
+        // We've moved past the end of the records and started
+        // trying to consume the index. This is the EOF from
+        // the client's perspective.
+        LOG.debug("Indexed position is itself an IndexSegment; false.");
+        return false;
+      }
+      LOG.debug("Aligned on record id=" + this.curEntryId);
+
+      this.claimedRecordLen = WritableUtils.readVLong(this.dataIn);
+      LOG.debug("Record has claimed length " + this.claimedRecordLen);
+      // We are now aligned on the start of the user's data.
+      this.isAligned = true;
+      return true;
+    }
+
+    /** {@inheritDoc} */
+    public boolean isRecordAvailable() {
+      return this.isAligned;
+    }
+
+    /** {@inheritDoc} */
+    public long getRecordLen() {
+      return this.claimedRecordLen;
+    }
+
+    /** {@inheritDoc} */
+    public long getRecordId() {
+      return this.curEntryId;
+    }
+
+    /** {@inheritDoc} */
+    public long getRecordOffset() {
+      return this.curRecordOffset;
+    }
+
+    /** {@inheritDoc} */
+    public InputStream readBlobRecord() throws IOException {
+      if (!isRecordAvailable()) {
+        // we're not currently aligned on a record-start.
+        // Try to get the next one.
+        if (!next()) {
+          // No more records available.
+          throw new EOFException("End of file reached.");
+        }
+      }
+
+      // Ensure any previously-open user record stream is closed.
+      closeUserStream();
+
+      // Mark this record as consumed.
+      this.isAligned = false;
+
+      // The length of the stream we can return to the user is
+      // the indexRecordLen minus the length of any per-record headers.
+      // That includes the RecordStartMark, the entryId, and the claimedLen.
+      long streamLen = this.indexRecordLen - RecordStartMark.START_MARK_LENGTH
+          - WritableUtils.getVIntSize(this.curEntryId)
+          - WritableUtils.getVIntSize(this.claimedRecordLen);
+      LOG.debug("Yielding stream to user with length " + streamLen);
+      this.userInputStream = new FixedLengthInputStream(this.dataIn, streamLen);
+      if (this.codec != null) {
+        // The user needs to decompress the data; wrap the InputStream.
+        decompressor.reset();
+        this.userInputStream = new DecompressorStream(
+            this.userInputStream, decompressor);
+      }
+      return this.userInputStream;
+    }
+
+    /** {@inheritDoc} */
+    public java.io.Reader readClobRecord() throws IOException {
+      // Get a handle to the binary reader and then wrap it.
+      InputStream is = readBlobRecord();
+      return new InputStreamReader(is);
+    }
+
+    /** {@inheritDoc} */
+    public void close() throws IOException {
+      closeUserStream();
+
+      if (null != dataIn) {
+        dataIn.close();
+        dataIn = null;
+      }
+
+      if (null != underlyingInput) {
+        underlyingInput.close();
+        underlyingInput = null;
+      }
+
+      this.isAligned = false;
+    }
+
+    /** {@inheritDoc} */
+    public boolean isClosed() {
+      return this.underlyingInput == null;
+    }
+  }
+
+  /**
+   * Creates a LobFile Reader configured to read from the specified file.
+   */
+  public static Reader open(Path p, Configuration conf) throws IOException {
+    FileSystem fs = p.getFileSystem(conf);
+    FileStatus [] stats = fs.listStatus(p);
+    if (null == stats || stats.length == 0) {
+      throw new IOException("Could not find file: " + p);
+    }
+    FSDataInputStream fis = fs.open(p);
+    DataInputStream dis = new DataInputStream(fis);
+    LobFileHeader header = new LobFileHeader(dis);
+    int version = header.getVersion();
+
+    if (version == 0) {
+      return new V0Reader(p, conf, header, dis, fis, stats[0].getLen());
+    } else {
+      throw new IOException("No reader available for LobFile version "  
+          + version);
+    }
+  }
+
+  /**
+   * Creates a LobFile Writer configured for uncompressed binary data.
+   * @param p the path to create.
+   * @param conf the configuration to use to interact with the filesystem.
+   */
+  public static Writer create(Path p, Configuration conf) throws IOException {
+    return create(p, conf, false);
+  }
+
+  /**
+   * Creates a LobFile Writer configured for uncompressed data.
+   * @param p the path to create.
+   * @param conf the configuration to use to interact with the filesystem.
+   * @param isCharData true if this is for CLOBs, false for BLOBs.
+   */
+  public static Writer create(Path p, Configuration conf, boolean isCharData)
+      throws IOException {
+    return create(p, conf, isCharData, null);
+  }
+
+  /**
+   * Creates a LobFile Writer.
+   * @param p the path to create.
+   * @param conf the configuration to use to interact with the filesystem.
+   * @param isCharData true if this is for CLOBs, false for BLOBs.
+   * @param codec the compression codec to use (or null for none).
+   */
+  public static Writer create(Path p, Configuration conf, boolean isCharData,
+      String codec) throws IOException {
+    return create(p, conf, isCharData, codec,
+        V0Writer.DEFAULT_MAX_SEGMENT_ENTRIES);
+  }
+
+  /**
+   * Creates a LobFile Writer.
+   * @param p the path to create.
+   * @param conf the configuration to use to interact with the filesystem.
+   * @param isCharData true if this is for CLOBs, false for BLOBs.
+   * @param codec the compression codec to use (or null for none).
+   * @param entriesPerSegment number of entries per index segment.
+   */
+  public static Writer create(Path p, Configuration conf, boolean isCharData,
+      String codec, int entriesPerSegment)
+      throws IOException {
+    return new V0Writer(p, conf, isCharData, codec, entriesPerSegment);
+  }
+}
+
diff --git a/src/java/org/apache/hadoop/sqoop/io/LobReaderCache.java b/src/java/org/apache/hadoop/sqoop/io/LobReaderCache.java
new file mode 100644
index 00000000..0cc79441
--- /dev/null
+++ b/src/java/org/apache/hadoop/sqoop/io/LobReaderCache.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop.io;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * A cache of open LobFile.Reader objects.
+ * This maps from filenames to the open Reader, if any.  This uses the
+ * Singleton pattern. While nothing prevents multiple LobReaderCache
+ * instances, it is most useful to have a single global cache. This cache is
+ * internally synchronized; only one thread can insert or retrieve a reader
+ * from the cache at a time. 
+ */
+public final class LobReaderCache {
+
+  public static final Log LOG = LogFactory.getLog(
+      LobReaderCache.class.getName());
+
+  private Map<Path, LobFile.Reader> readerMap;
+
+  private LobReaderCache() {
+    this.readerMap = new TreeMap<Path, LobFile.Reader>();
+  }
+
+  private static final LobReaderCache CACHE;
+  static {
+    CACHE = new LobReaderCache();
+  }
+
+  /**
+   * @return the singleton LobReaderCache instance.
+   */
+  public static LobReaderCache getCache() {
+    return CACHE;
+  }
+
+  /**
+   * Created a fully-qualified path object.
+   * @param path the path to fully-qualify with its fs URI.
+   * @param conf the current Hadoop FS configuration.
+   * @return a new path representing the same location as the input 'path',
+   * but with a fully-qualified URI.
+   */
+  public static Path qualify(Path path, Configuration conf)
+      throws IOException {
+    if (null == path) {
+      return null;
+    }
+
+    FileSystem fs = path.getFileSystem(conf);
+    if (null == fs) {
+      fs = FileSystem.get(conf);
+    }
+    return path.makeQualified(fs);
+  }
+  
+  /**
+   * Open a LobFile for read access, returning a cached reader if one is
+   * available, or a new reader otherwise.
+   * @param path the path to the LobFile to open
+   * @param conf the configuration to use to access the FS.
+   * @throws IOException if there's an error opening the file.
+   */
+  public LobFile.Reader get(Path path, Configuration conf)
+      throws IOException {
+
+    LobFile.Reader reader = null;
+    Path canonicalPath = qualify(path, conf);
+    // Look up an entry in the cache.
+    synchronized(this) {
+      reader = readerMap.remove(canonicalPath);
+    }
+
+    if (null != reader && !reader.isClosed()) {
+      // Cache hit. return it.
+      LOG.debug("Using cached reader for " + canonicalPath);
+      return reader;
+    }
+
+    // Cache miss; open the file.
+    LOG.debug("No cached reader available for " + canonicalPath);
+    return LobFile.open(path, conf);
+  }
+
+  /**
+   * Return a reader back to the cache. If there's already a reader for
+   * this path, then the current reader is closed.
+   * @param reader the opened reader. Any record-specific subreaders should be
+   * closed.
+   * @throws IOException if there's an error accessing the path's filesystem.
+   */
+  public void recycle(LobFile.Reader reader) throws IOException {
+    Path canonicalPath = reader.getPath();
+
+    // Check if the cache has a reader for this path already. If not, add this.
+    boolean cached = false;
+    synchronized(this) {
+      if (readerMap.get(canonicalPath) == null) {
+        LOG.debug("Caching reader for path: " + canonicalPath);
+        readerMap.put(canonicalPath, reader);
+        cached = true;
+      }
+    }
+
+    if (!cached) {
+      LOG.debug("Reader already present for path: " + canonicalPath
+          + "; closing.");
+      reader.close();
+    }
+  }
+
+  @Override
+  protected synchronized void finalize() throws Throwable {
+    for (LobFile.Reader r : readerMap.values()) {
+      r.close();
+    }
+
+    super.finalize();
+  }
+}
+
diff --git a/src/java/org/apache/hadoop/sqoop/io/UnsupportedCodecException.java b/src/java/org/apache/hadoop/sqoop/io/UnsupportedCodecException.java
new file mode 100644
index 00000000..1689c5b0
--- /dev/null
+++ b/src/java/org/apache/hadoop/sqoop/io/UnsupportedCodecException.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop.io;
+
+import java.io.IOException;
+
+/**
+ * Thrown when a compression codec cannot be recognized.
+ */
+public class UnsupportedCodecException extends IOException {
+  public UnsupportedCodecException() {
+    super("UnsupportedCodecException");
+  }
+
+  public UnsupportedCodecException(String msg) {
+    super(msg);
+  }
+
+  public UnsupportedCodecException(Throwable cause) {
+    super(cause);
+  }
+}
diff --git a/src/java/org/apache/hadoop/sqoop/lib/BlobRef.java b/src/java/org/apache/hadoop/sqoop/lib/BlobRef.java
index e9bd0304..487b1ae6 100644
--- a/src/java/org/apache/hadoop/sqoop/lib/BlobRef.java
+++ b/src/java/org/apache/hadoop/sqoop/lib/BlobRef.java
@@ -35,153 +35,70 @@
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.sqoop.io.LobFile;
+import org.apache.hadoop.sqoop.io.LobReaderCache;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 /**
- * BlobRef is a wrapper that holds a Blob either directly, or a
- * reference to a file that holds the blob data.
+ * BlobRef is a wrapper that holds a BLOB either directly, or a
+ * reference to a file that holds the BLOB data.
  */
-public class BlobRef implements Writable {
+public class BlobRef extends LobRef<byte[], BytesWritable, InputStream> {
 
   public static final Log LOG = LogFactory.getLog(BlobRef.class.getName());
 
-  public BlobRef(byte [] bytes) {
-    this.fileName = null;
-    this.data = new BytesWritable(bytes);
-  }
-
   public BlobRef() {
-    this.fileName = null;
-    this.data = null;
+    super();
   }
 
-  public BlobRef(String file) {
-    this.fileName = file;
-    this.data = null;
-  }
-
-  // If the data is 'small', it's held directly, here.
-  private BytesWritable data;
-
-  // If there data is too large, it's written into a file
-  // whose path (relative to the rest of the dataset) is recorded here.
-  // This takes precedence if this value is non-null.
-  private String fileName;
-
-  /**
-   * @return true if the BLOB data is in an external file; false if
-   * it materialized inline.
-   */
-  public boolean isExternal() {
-    return fileName != null;
+  public BlobRef(byte [] bytes) {
+    super(new BytesWritable(bytes));
   }
 
   /**
-   * Convenience method to access #getDataStream(Configuration, Path)
-   * from within a map task that read this BlobRef from a file-based
-   * InputSplit.
-   * @param mapContext the Mapper.Context instance that encapsulates
-   * the current map task.
-   * @return an InputStream to access the BLOB data.
-   * @throws IllegalArgumentException if it cannot find the source
-   * path for this BLOB based on the MapContext.
-   * @throws IOException if it could not read the BLOB from external storage.
+   * Initialize a BlobRef to an external BLOB.
+   * @param file the filename to the BLOB. May be relative to the job dir.
+   * @param offset the offset (in bytes) into the LobFile for this record.
+   * @param length the length of the record in bytes.
    */
-  public InputStream getDataStream(Mapper.Context mapContext)
-      throws IllegalArgumentException, IOException {
-    InputSplit split = mapContext.getInputSplit();
-    if (split instanceof FileSplit) {
-      Path basePath = ((FileSplit) split).getPath().getParent();
-      return getDataStream(mapContext.getConfiguration(),
-        basePath);
-    } else {
-      throw new IllegalArgumentException(
-          "Could not ascertain BLOB base path from MapContext.");
-    }
+  public BlobRef(String file, long offset, long length) {
+    super(file, offset, length);
   }
 
-  /**
-   * Get access to the BLOB data itself.
-   * This method returns an InputStream-based representation of the
-   * BLOB data, accessing the filesystem for external BLOB storage
-   * as necessary.
-   * @param conf the Configuration used to access the filesystem
-   * @param basePath the base directory where the table records are
-   * stored.
-   * @return an InputStream used to read the BLOB data.
-   * @throws IOException if it could not read the BLOB from external storage.
-   */
-  public InputStream getDataStream(Configuration conf, Path basePath)
+  @Override
+  protected InputStream getExternalSource(LobFile.Reader reader)
       throws IOException {
-    if (isExternal()) {
-      // use external storage.
-      FileSystem fs = FileSystem.get(conf);
-      return fs.open(new Path(basePath, fileName));
-    } else {
-      return new ByteArrayInputStream(data.getBytes());
-    }
+    return reader.readBlobRecord();
   }
 
+  @Override
+  protected InputStream getInternalSource(BytesWritable data) {
+    return new ByteArrayInputStream(data.getBytes());
+  }
 
-  public byte [] getData() {
-    if (isExternal()) {
-      throw new RuntimeException(
-          "External BLOBs must be read via getDataStream()");
-    }
-
+  @Override
+  protected byte [] getInternalData(BytesWritable data) {
     return data.getBytes();
   }
 
   @Override
-  public String toString() {
-    if (isExternal()) {
-      return "externalBlob(" + fileName + ")";
-    } else {
-      return data.toString();
+  public void readFieldsInternal(DataInput in) throws IOException {
+    // For internally-stored BLOBs, the data is a BytesWritable
+    // containing the actual data.
+
+    if (null == this.data) {
+      this.data = new BytesWritable();
     }
+    this.data.readFields(in);
   }
 
   @Override
-  public void readFields(DataInput in) throws IOException {
-    // The serialization format for this object is:
-    // boolean isExternal
-    // if true, the next field is a String containing the file name.
-    // if false, the next field is a BytesWritable containing the
-    // actual data.
-
-    boolean isExternal = in.readBoolean();
-    if (isExternal) {
-      this.data = null;
-      this.fileName = Text.readString(in);
-    } else {
-      if (null == this.data) {
-        this.data = new BytesWritable();
-      }
-      this.data.readFields(in);
-      this.fileName = null;
-    }
+  public void writeInternal(DataOutput out) throws IOException {
+    data.write(out);
   }
 
-  @Override
-  public void write(DataOutput out) throws IOException {
-    out.writeBoolean(isExternal());
-    if (isExternal()) {
-      Text.writeString(out, fileName);
-    } else {
-      data.write(out);
-    }
-  }
-
-  private static final ThreadLocal<Matcher> EXTERNAL_MATCHER =
-    new ThreadLocal<Matcher>() {
-      @Override protected Matcher initialValue() {
-        Pattern externalPattern = Pattern.compile("externalBlob\\((.*)\\)");
-        return externalPattern.matcher("");
-      }
-    };
-
   /**
    * Create a BlobRef based on parsed data from a line of text.
    * This only operates correctly on external blobs; inline blobs are simply
@@ -192,15 +109,18 @@ public void write(DataOutput out) throws IOException {
    * an empty BlobRef if the data to be parsed is actually inline.
    */
   public static BlobRef parse(String inputString) {
-    // If inputString is of the form 'externalBlob(%s)', then this is an
-    // external BLOB stored at the filename indicated by '%s'. Otherwise,
-    // it is an inline BLOB, which we don't support parsing of.
+    // If inputString is of the form 'externalLob(lf,%s,%d,%d)', then this is
+    // an external BLOB stored at the LobFile indicated by '%s' with the next
+    // two arguments representing its offset and length in the file.
+    // Otherwise, it is an inline BLOB, which we don't support parsing of.
 
     Matcher m = EXTERNAL_MATCHER.get();
     m.reset(inputString);
     if (m.matches()) {
-      // Extract the filename component from the string.
-      return new BlobRef(m.group(1));
+      // This is a LobFile. Extract the filename, offset and len from the
+      // matcher.
+      return new BlobRef(m.group(1), Long.valueOf(m.group(2)),
+          Long.valueOf(m.group(3)));
     } else {
       // This is inline BLOB string data.
       LOG.warn(
diff --git a/src/java/org/apache/hadoop/sqoop/lib/ClobRef.java b/src/java/org/apache/hadoop/sqoop/lib/ClobRef.java
index 43211d69..782e2ae3 100644
--- a/src/java/org/apache/hadoop/sqoop/lib/ClobRef.java
+++ b/src/java/org/apache/hadoop/sqoop/lib/ClobRef.java
@@ -35,149 +35,58 @@
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.sqoop.io.LobFile;
 
 /**
- * ClobRef is a wrapper that holds a Clob either directly, or a
- * reference to a file that holds the clob data.
+ * ClobRef is a wrapper that holds a CLOB either directly, or a
+ * reference to a file that holds the CLOB data.
  */
-public class ClobRef implements Writable {
-
-  public ClobRef(String chars) {
-    this.fileName = null;
-    this.data = chars;
-  }
+public class ClobRef extends LobRef<String, String, Reader> {
 
   public ClobRef() {
-    this.fileName = null;
-    this.data = null;
+    super();
+  }
+
+  public ClobRef(String chars) {
+    super(chars);
   }
 
   /**
    * Initialize a clobref to an external CLOB.
    * @param file the filename to the CLOB. May be relative to the job dir.
-   * @param ignored is not used; this just differentiates this constructor
-   * from ClobRef(String chars).
+   * @param offset the offset (in bytes) into the LobFile for this record.
+   * @param length the length of the record in characters.
    */
-  public ClobRef(String file, boolean ignored) {
-    this.fileName = file;
-    this.data = null;
+  public ClobRef(String file, long offset, long length) {
+    super(file, offset, length);
   }
 
-  // If the data is 'small', it's held directly, here.
-  private String data;
-
-  // If there data is too large, it's written into a file
-  // whose path (relative to the rest of the dataset) is recorded here.
-  // This takes precedence if this value is non-null.
-  private String fileName;
-
-  /**
-   * @return true if the CLOB data is in an external file; false if
-   * it is materialized inline.
-   */
-  public boolean isExternal() {
-    return fileName != null;
-  }
-
-  /**
-   * Convenience method to access #getDataReader(Configuration, Path)
-   * from within a map task that read this ClobRef from a file-based
-   * InputSplit.
-   * @param mapContext the Mapper.Context instance that encapsulates
-   * the current map task.
-   * @return a Reader to access the CLOB data.
-   * @throws IllegalArgumentException if it cannot find the source
-   * path for this CLOB based on the MapContext.
-   * @throws IOException if it could not read the CLOB from external storage.
-   */
-  public Reader getDataReader(Mapper.Context mapContext)
-      throws IllegalArgumentException, IOException {
-    InputSplit split = mapContext.getInputSplit();
-    if (split instanceof FileSplit) {
-      Path basePath = ((FileSplit) split).getPath().getParent();
-      return getDataReader(mapContext.getConfiguration(),
-        basePath);
-    } else {
-      throw new IllegalArgumentException(
-          "Could not ascertain CLOB base path from MapContext.");
-    }
-  }
-
-  /**
-   * Get access to the CLOB data itself.
-   * This method returns a Reader-based representation of the
-   * CLOB data, accessing the filesystem for external CLOB storage
-   * as necessary.
-   * @param conf the Configuration used to access the filesystem
-   * @param basePath the base directory where the table records are
-   * stored.
-   * @return a Reader used to read the CLOB data.
-   * @throws IOException if it could not read the CLOB from external storage.
-   */
-  public Reader getDataReader(Configuration conf, Path basePath)
+  @Override
+  protected Reader getExternalSource(LobFile.Reader reader)
       throws IOException {
-    if (isExternal()) {
-      // use external storage.
-      FileSystem fs = FileSystem.get(conf);
-      return new InputStreamReader(fs.open(new Path(basePath, fileName)));
-    } else {
-      return new StringReader(data);
-    }
+    return reader.readClobRecord();
   }
 
   @Override
-  /**
-   * @return a string representation of the ClobRef. If this is an
-   * inline clob (isExternal() returns false), it will contain the
-   * materialized data. Otherwise it returns a description of the
-   * reference. To ensure access to the data itself,
-   * {@see #getDataReader(Configuration,Path)}.
-   */
-  public String toString() {
-    if (isExternal()) {
-      return "externalClob(" + fileName + ")";
-    } else {
-      return data;
-    }
+  protected Reader getInternalSource(String data) {
+    return new StringReader(data);
   }
 
   @Override
-  public void readFields(DataInput in) throws IOException {
-    // The serialization format for this object is:
-    // boolean isExternal
-    // if true, the next field is a String containing the external file name.
-    // if false, the next field is String containing the actual data.
-
-    boolean isIndirect = in.readBoolean();
-    if (isIndirect) {
-      this.fileName = Text.readString(in);
-      this.data = null;
-    } else {
-      this.fileName = null;
-      this.data = Text.readString(in);
-    }
+  protected String getInternalData(String data) {
+    return data;
   }
 
   @Override
-  public void write(DataOutput out) throws IOException {
-    boolean isIndirect = isExternal();
-    out.writeBoolean(isIndirect);
-    if (isIndirect) {
-      Text.writeString(out, fileName);
-    } else {
-      Text.writeString(out, data);
-    }
+  public void readFieldsInternal(DataInput in) throws IOException {
+    // For internally-stored clobs, the data is written as UTF8 Text.
+    this.data = Text.readString(in);
   }
 
-  // A pattern matcher which can recognize external CLOB data
-  // vs. an inline CLOB string.
-  private static final ThreadLocal<Matcher> EXTERNAL_MATCHER =
-    new ThreadLocal<Matcher>() {
-      @Override protected Matcher initialValue() {
-        Pattern externalPattern = Pattern.compile("externalClob\\((.*)\\)");
-        return externalPattern.matcher("");
-      }
-    };
+  @Override
+  public void writeInternal(DataOutput out) throws IOException {
+    Text.writeString(out, data);
+  }
 
   /**
    * Create a ClobRef based on parsed data from a line of text.
@@ -185,15 +94,18 @@ public void write(DataOutput out) throws IOException {
    * @return a ClobRef to the given data.
    */
   public static ClobRef parse(String inputString) {
-    // If inputString is of the form 'externalClob(%s)', then this is an
-    // external CLOB stored at the filename indicated by '%s'. Otherwise,
-    // it is an inline CLOB.
+    // If inputString is of the form 'externalLob(lf,%s,%d,%d)', then this is
+    // an external CLOB stored at the LobFile indicated by '%s' with the next
+    // two arguments representing its offset and length in the file.
+    // Otherwise, it is an inline CLOB, which we read as-is.
 
     Matcher m = EXTERNAL_MATCHER.get();
     m.reset(inputString);
     if (m.matches()) {
-      // Extract the filename component from the string.
-      return new ClobRef(m.group(1), true);
+      // This is a LobFile. Extract the filename, offset and len from the
+      // matcher.
+      return new ClobRef(m.group(1), Long.valueOf(m.group(2)),
+          Long.valueOf(m.group(3)));
     } else {
       // This is inline CLOB string data.
       return new ClobRef(inputString);
diff --git a/src/java/org/apache/hadoop/sqoop/lib/LargeObjectLoader.java b/src/java/org/apache/hadoop/sqoop/lib/LargeObjectLoader.java
index 7ec1c9cb..a31fb0b3 100644
--- a/src/java/org/apache/hadoop/sqoop/lib/LargeObjectLoader.java
+++ b/src/java/org/apache/hadoop/sqoop/lib/LargeObjectLoader.java
@@ -20,6 +20,8 @@
 
 import java.io.BufferedOutputStream;
 import java.io.BufferedWriter;
+import java.io.Closeable;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -40,6 +42,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.sqoop.io.LobFile;
 
 /**
  * Contains a set of methods which can read db columns from a ResultSet into
@@ -51,9 +54,9 @@
  * However, its lifetime is limited to the current TaskInputOutputContext's
  * life.
  */
-public class LargeObjectLoader {
+public class LargeObjectLoader implements Closeable {
 
-  // Currently, cap BLOB/CLOB objects at 16 MB until we can use external storage.
+  // Spill to external storage for BLOB/CLOB objects > 16 MB.
   public final static long DEFAULT_MAX_LOB_LENGTH = 16 * 1024 * 1024;
 
   public final static String MAX_INLINE_LOB_LEN_KEY =
@@ -63,6 +66,10 @@ public class LargeObjectLoader {
   private Path workPath;
   private FileSystem fs;
 
+  // Handles to the open BLOB / CLOB file writers. 
+  private LobFile.Writer curBlobWriter;
+  private LobFile.Writer curClobWriter;
+
   // Counter that is used with the current task attempt id to
   // generate unique LOB file names.
   private long nextLobFileId = 0;
@@ -77,19 +84,99 @@ public LargeObjectLoader(Configuration conf, Path workPath)
     this.conf = conf;
     this.workPath = workPath;
     this.fs = FileSystem.get(conf);
+    this.curBlobWriter = null;
+    this.curClobWriter = null;
+  }
+
+  @Override
+  protected synchronized void finalize() throws Throwable {
+    close();
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (null != curBlobWriter) {
+      curBlobWriter.close();
+      curBlobWriter = null;
+    }
+
+    if (null != curClobWriter) {
+      curClobWriter.close();
+      curClobWriter = null;
+    }
   }
 
   /**
    * @return a filename to use to put an external LOB in.
    */
   private String getNextLobFileName() {
-    String file = "_lob/obj_" + TaskId.get(conf, "unknown_task_id")
-        + nextLobFileId;
+    String file = "_lob/large_obj_" + TaskId.get(conf, "unknown_task_id")
+        + nextLobFileId + ".lob";
     nextLobFileId++;
 
     return file;
   }
 
+  /**
+   * Calculates a path to a new LobFile object, creating any
+   * missing directories.
+   * @return a Path to a LobFile to write
+   */
+  private Path getNextLobFilePath() throws IOException {
+    Path p = new Path(workPath, getNextLobFileName());
+    Path parent = p.getParent();
+    if (!fs.exists(parent)) {
+      fs.mkdirs(parent);
+    }
+
+    return p;
+  }
+
+  /**
+   * @return the current LobFile writer for BLOBs, creating one if necessary.
+   */
+  private LobFile.Writer getBlobWriter() throws IOException {
+    if (null == this.curBlobWriter) {
+      this.curBlobWriter = LobFile.create(getNextLobFilePath(), conf, false);
+    }
+
+    return this.curBlobWriter;
+  }
+
+  /**
+   * @return the current LobFile writer for CLOBs, creating one if necessary.
+   */
+  private LobFile.Writer getClobWriter() throws IOException {
+    if (null == this.curClobWriter) {
+      this.curClobWriter = LobFile.create(getNextLobFilePath(), conf, true);
+    }
+
+    return this.curClobWriter;
+  }
+
+  /**
+   * Returns the path being written to by a given LobFile.Writer, relative
+   * to the working directory of this LargeObjectLoader.
+   * @param w the LobFile.Writer whose path should be examined.
+   * @return the path this is writing to, relative to the current working dir.
+   */
+  private String getRelativePath(LobFile.Writer w) {
+    Path writerPath = w.getPath();
+    
+    String writerPathStr = writerPath.toString();
+    String workPathStr = workPath.toString();
+    if (!workPathStr.endsWith(File.separator)) {
+      workPathStr = workPathStr + File.separator;
+    }
+
+    if (writerPathStr.startsWith(workPathStr)) {
+      return writerPathStr.substring(workPathStr.length());
+    }
+
+    // Outside the working dir; return the whole thing.
+    return writerPathStr;
+  }
+
   /**
    * Copies all character data from the provided Reader to the provided
    * Writer. Does not close handles when it's done.
@@ -155,27 +242,16 @@ public BlobRef readBlobRef(int colNum, ResultSet r)
       return null;
     } else if (b.length() > maxInlineLobLen) {
       // Deserialize very large BLOBs into separate files.
-      String fileName = getNextLobFileName();
-      Path p = new Path(workPath, fileName);
+      long len = b.length();
+      LobFile.Writer lobWriter = getBlobWriter();
 
-      Path parent = p.getParent();
-      if (!fs.exists(parent)) {
-        fs.mkdirs(parent);
-      }
-
-      BufferedOutputStream bos = null;
+      long recordOffset = lobWriter.tell();
       InputStream is = null;
-      OutputStream os = fs.create(p);
+      OutputStream os = lobWriter.writeBlobRecord(len);
       try {
-        bos = new BufferedOutputStream(os);
         is = b.getBinaryStream();
-        copyAll(is, bos);
+        copyAll(is, os);
       } finally {
-        if (null != bos) {
-          bos.close();
-          os = null; // os is now closed.
-        }
-
         if (null != os) {
           os.close();
         }
@@ -183,9 +259,12 @@ public BlobRef readBlobRef(int colNum, ResultSet r)
         if (null != is) {
           is.close();
         }
+
+        // Mark the record as finished.
+        lobWriter.finishRecord();
       }
 
-      return new BlobRef(fileName);
+      return new BlobRef(getRelativePath(curBlobWriter), recordOffset, len);
     } else {
       // This is a 1-based array.
       return new BlobRef(b.getBytes(1, (int) b.length()));
@@ -215,37 +294,29 @@ public ClobRef readClobRef(int colNum, ResultSet r)
       return null;
     } else if (c.length() > maxInlineLobLen) {
       // Deserialize large CLOB into separate file.
-      String fileName = getNextLobFileName();
-      Path p = new Path(workPath, fileName);
+      long len = c.length();
+      LobFile.Writer lobWriter = getClobWriter();
 
-      Path parent = p.getParent();
-      if (!fs.exists(parent)) {
-        fs.mkdirs(parent);
-      }
-
-      BufferedWriter w = null;
+      long recordOffset = lobWriter.tell();
       Reader reader = null;
-      OutputStream os = fs.create(p);
+      Writer w = lobWriter.writeClobRecord(len);
       try {
-        w = new BufferedWriter(new OutputStreamWriter(os));
         reader = c.getCharacterStream();
         copyAll(reader, w);
       } finally {
         if (null != w) {
           w.close();
-          os = null; // os is now closed.
-        }
-
-        if (null != os) {
-          os.close();
         }
 
         if (null != reader) {
           reader.close();
         }
+
+        // Mark the record as finished.
+        lobWriter.finishRecord();
       }
 
-      return new ClobRef(fileName, true);
+      return new ClobRef(getRelativePath(lobWriter), recordOffset, len);
     } else {
       // This is a 1-based array.
       return new ClobRef(c.getSubString(1, (int) c.length()));
diff --git a/src/java/org/apache/hadoop/sqoop/lib/LobRef.java b/src/java/org/apache/hadoop/sqoop/lib/LobRef.java
new file mode 100644
index 00000000..edf9f94f
--- /dev/null
+++ b/src/java/org/apache/hadoop/sqoop/lib/LobRef.java
@@ -0,0 +1,295 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop.lib;
+
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.sqoop.io.LobFile;
+import org.apache.hadoop.sqoop.io.LobReaderCache;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Abstract base class that holds a reference to a Blob or a Clob.
+ * DATATYPE is the type being held (e.g., a byte array).
+ * CONTAINERTYPE is the type used to hold this data (e.g., BytesWritable).
+ * ACCESSORTYPE is the type used to access this data in a streaming fashion
+ *   (either an InputStream or a Reader).
+ */
+public abstract class LobRef<DATATYPE, CONTAINERTYPE, ACCESSORTYPE>
+    implements Closeable, Writable {
+
+  public static final Log LOG = LogFactory.getLog(LobRef.class.getName());
+
+  protected LobRef() {
+    this.fileName = null;
+    this.offset = 0;
+    this.length = 0;
+
+    this.data = null;
+  }
+
+  protected LobRef(CONTAINERTYPE container) {
+    this.fileName = null;
+    this.offset = 0;
+    this.length = 0;
+
+    this.data = container;
+  }
+
+  protected LobRef(String file, long offset, long length) {
+    this.fileName = file;
+    this.offset = offset;
+    this.length = length;
+
+    this.data = null;
+  }
+
+  // If the data is 'small', it's held directly, here.
+  protected CONTAINERTYPE data;
+
+  // If there data is too large to materialize fully, it's written into a file
+  // whose path (relative to the rest of the dataset) is recorded here. This
+  // takes precedence if the value fof fileName is non-null. These records are
+  // currently written into LobFile-formatted files, which hold multiple
+  // records. The starting offset and length of the record are recorded here
+  // as well.
+  private String fileName;
+  private long offset;
+  private long length;
+
+  // If we've opened a LobFile object, track our reference to it here.
+  protected LobFile.Reader reader;
+
+  @Override
+  protected synchronized void finalize() throws Throwable {
+    close();
+  }
+
+  public void close() throws IOException {
+    // Discard any open LobReader.
+    if (null != this.reader) {
+      LobReaderCache.getCache().recycle(this.reader);
+    }
+  }
+
+  /**
+   * @return true if the LOB data is in an external file; false if
+   * it materialized inline.
+   */
+  public boolean isExternal() {
+    return fileName != null;
+  }
+
+  /**
+   * Convenience method to access #getDataStream(Configuration, Path)
+   * from within a map task that read this LobRef from a file-based
+   * InputSplit.
+   * @param mapContext the Mapper.Context instance that encapsulates
+   * the current map task.
+   * @return an object that lazily streams the record to the client.
+   * @throws IllegalArgumentException if it cannot find the source
+   * path for this LOB based on the MapContext.
+   * @throws IOException if it could not read the LOB from external storage.
+   */
+  public ACCESSORTYPE getDataStream(Mapper.Context mapContext)
+      throws IllegalArgumentException, IOException {
+    InputSplit split = mapContext.getInputSplit();
+    if (split instanceof FileSplit) {
+      Path basePath = ((FileSplit) split).getPath().getParent();
+      return getDataStream(mapContext.getConfiguration(),
+        basePath);
+    } else {
+      throw new IllegalArgumentException(
+          "Could not ascertain LOB base path from MapContext.");
+    }
+  }
+
+  /**
+   * Get access to the LOB data itself.
+   * This method returns a lazy reader of the LOB data, accessing the
+   * filesystem for external LOB storage as necessary.
+   * @param conf the Configuration used to access the filesystem
+   * @param basePath the base directory where the table records are
+   * stored.
+   * @return an object that lazily streams the record to the client.
+   * @throws IOException if it could not read the LOB from external storage.
+   */
+  public ACCESSORTYPE getDataStream(Configuration conf, Path basePath)
+      throws IOException {
+    if (isExternal()) {
+      // Read from external storage.
+      Path pathToRead = LobReaderCache.qualify(
+          new Path(basePath, fileName), conf);
+      LOG.debug("Retreving data stream from external path: " + pathToRead);
+      if (reader != null) {
+        // We already have a reader open to a LobFile. Is it the correct file?
+        if (!pathToRead.equals(reader.getPath())) {
+          // No. Close this reader and get the correct one.
+          LOG.debug("Releasing previous external reader for "
+              + reader.getPath());
+          LobReaderCache.getCache().recycle(reader);
+          reader = LobReaderCache.getCache().get(pathToRead, conf);
+        }
+      } else {
+        reader = LobReaderCache.getCache().get(pathToRead, conf);
+      }
+
+      // We now have a LobFile.Reader associated with the correct file. Get to
+      // the correct offset and return an InputStream/Reader to the user.
+      if (reader.tell() != offset) {
+        LOG.debug("Seeking to record start offset " + offset);
+        reader.seek(offset);
+      }
+
+      if (!reader.next()) {
+        throw new IOException("Could not locate record at " + pathToRead
+            + ":" + offset);
+      }
+
+      return getExternalSource(reader);
+    } else {
+      // This data is already materialized in memory; wrap it and return.
+      return getInternalSource(data);
+    }
+  }
+
+  /**
+   * Using the LobFile reader, get an accessor InputStream or Reader to the
+   * underlying data.
+   */
+  protected abstract ACCESSORTYPE getExternalSource(LobFile.Reader reader)
+      throws IOException;
+
+  /**
+   * Wrap the materialized data in an InputStream or Reader.
+   */
+  protected abstract ACCESSORTYPE getInternalSource(CONTAINERTYPE data);
+
+  /**
+   * @return the materialized data itself.
+   */
+  protected abstract DATATYPE getInternalData(CONTAINERTYPE data);
+
+  public DATATYPE getData() {
+    if (isExternal()) {
+      throw new RuntimeException(
+          "External LOBs must be read via getDataStream()");
+    }
+
+    return getInternalData(data);
+  }
+
+  @Override
+  public String toString() {
+    if (isExternal()) {
+      return "externalLob(lf," + fileName + "," + Long.toString(offset)
+          + "," + Long.toString(length) + ")";
+    } else {
+      return data.toString();
+    }
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    // The serialization format for this object is:
+    // boolean isExternal
+    // if true, then:
+    //   a string identifying the external storage type
+    //   and external-storage-specific data.
+    // if false, then we use readFieldsInternal() to allow BlobRef/ClobRef
+    // to serialize as it sees fit.
+    //
+    // Currently the only external storage supported is LobFile, identified
+    // by the string "lf". This serializes with the filename (as a string),
+    // followed by a long-valued offset and a long-valued length.
+
+    boolean isExternal = in.readBoolean();
+    if (isExternal) {
+      this.data = null;
+
+      String storageType = Text.readString(in);
+      if (!storageType.equals("lf")) {
+        throw new IOException("Unsupported external LOB storage code: "
+            + storageType);
+      }
+
+      // Storage type "lf" is LobFile: filename, offset, length.
+      this.fileName = Text.readString(in);
+      this.offset = in.readLong();
+      this.length = in.readLong();
+    } else {
+      readFieldsInternal(in);
+
+      this.fileName = null;
+      this.offset = 0;
+      this.length = 0;
+    }
+  }
+
+  /**
+   * Perform the readFields() operation on a fully-materializable record.
+   * @param in the DataInput to deserialize from.
+   */
+  protected abstract void readFieldsInternal(DataInput in) throws IOException;
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeBoolean(isExternal());
+    if (isExternal()) {
+      Text.writeString(out, "lf"); // storage type "lf" for LobFile.
+      Text.writeString(out, fileName);
+      out.writeLong(offset);
+      out.writeLong(length);
+    } else {
+      writeInternal(out);
+    }
+  }
+
+  /**
+   * Perform the write() operation on a fully-materializable record.
+   * @param out the DataOutput to deserialize to.
+   */
+  protected abstract void writeInternal(DataOutput out) throws IOException;
+
+  protected static final ThreadLocal<Matcher> EXTERNAL_MATCHER =
+    new ThreadLocal<Matcher>() {
+      @Override protected Matcher initialValue() {
+        Pattern externalPattern = Pattern.compile(
+            "externalLob\\(lf,(.*),([0-9]+),([0-9]+)\\)");
+        return externalPattern.matcher("");
+      }
+    };
+}
+
diff --git a/src/java/org/apache/hadoop/sqoop/mapreduce/SequenceFileImportMapper.java b/src/java/org/apache/hadoop/sqoop/mapreduce/SequenceFileImportMapper.java
index c4c7b7dc..3f337b05 100644
--- a/src/java/org/apache/hadoop/sqoop/mapreduce/SequenceFileImportMapper.java
+++ b/src/java/org/apache/hadoop/sqoop/mapreduce/SequenceFileImportMapper.java
@@ -33,18 +33,33 @@
 public class SequenceFileImportMapper
     extends AutoProgressMapper<LongWritable, SqoopRecord, LongWritable, SqoopRecord> {
 
+  private LargeObjectLoader lobLoader;
+
+  @Override
+  protected void setup(Context context) throws IOException, InterruptedException {
+    this.lobLoader = new LargeObjectLoader(context.getConfiguration(),
+        FileOutputFormat.getWorkOutputPath(context));
+  }
+
+  @Override
   public void map(LongWritable key, SqoopRecord val, Context context)
       throws IOException, InterruptedException {
 
     try {
       // Loading of LOBs was delayed until we have a Context.
-      val.loadLargeObjects(new LargeObjectLoader(context.getConfiguration(),
-          FileOutputFormat.getWorkOutputPath(context)));
+      val.loadLargeObjects(lobLoader);
     } catch (SQLException sqlE) {
       throw new IOException(sqlE);
     }
 
     context.write(key, val);
   }
+
+  @Override
+  protected void cleanup(Context context) throws IOException {
+    if (null != lobLoader) {
+      lobLoader.close();
+    }
+  }
 }
 
diff --git a/src/java/org/apache/hadoop/sqoop/mapreduce/TextImportMapper.java b/src/java/org/apache/hadoop/sqoop/mapreduce/TextImportMapper.java
index f24a37ab..57042c82 100644
--- a/src/java/org/apache/hadoop/sqoop/mapreduce/TextImportMapper.java
+++ b/src/java/org/apache/hadoop/sqoop/mapreduce/TextImportMapper.java
@@ -36,18 +36,25 @@ public class TextImportMapper
     extends AutoProgressMapper<LongWritable, SqoopRecord, Text, NullWritable> {
 
   private Text outkey;
+  private LargeObjectLoader lobLoader;
 
   public TextImportMapper() {
     outkey = new Text();
   }
 
+  @Override
+  protected void setup(Context context) throws IOException, InterruptedException {
+    this.lobLoader = new LargeObjectLoader(context.getConfiguration(),
+        FileOutputFormat.getWorkOutputPath(context));
+  }
+
+  @Override
   public void map(LongWritable key, SqoopRecord val, Context context)
       throws IOException, InterruptedException {
 
     try {
       // Loading of LOBs was delayed until we have a Context.
-      val.loadLargeObjects(new LargeObjectLoader(context.getConfiguration(),
-          FileOutputFormat.getWorkOutputPath(context)));
+      val.loadLargeObjects(lobLoader);
     } catch (SQLException sqlE) {
       throw new IOException(sqlE);
     }
@@ -55,5 +62,12 @@ public void map(LongWritable key, SqoopRecord val, Context context)
     outkey.set(val.toString());
     context.write(outkey, NullWritable.get());
   }
+
+  @Override
+  protected void cleanup(Context context) throws IOException {
+    if (null != lobLoader) {
+      lobLoader.close();
+    }
+  }
 }
 
diff --git a/src/perftest/LobFilePerfTest.java b/src/perftest/LobFilePerfTest.java
new file mode 100644
index 00000000..8d080cbe
--- /dev/null
+++ b/src/perftest/LobFilePerfTest.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.sqoop.io.*;
+
+/**
+ * A simple benchmark to performance test LobFile reader/writer speed.
+ * Writes out 10 GB of data to the local disk and then reads it back.
+ * Run with:
+ * HADOOP_OPTS=-agentlib:hprof=cpu=samples src/scripts/run-perftest.sh LobFilePerfTest
+ */
+public class LobFilePerfTest {
+
+  long recordLen = 20 * 1024 * 1024; // 20 MB records
+  int numRecords = 500;
+  Configuration conf;
+  Path p;
+  long startTime;
+  byte [] record;
+
+  public LobFilePerfTest() {
+    conf = new Configuration();
+    conf.set("fs.default.name", "file:///");
+    p = new Path("foo.lob");
+  }
+
+
+  private void startTiming(String s) {
+    System.out.println(s);
+    startTime = System.currentTimeMillis();
+  }
+
+  private void stopTiming() {
+    long finishTime = System.currentTimeMillis();
+    long delta = finishTime - startTime;
+    System.out.println("Finished. Time elapsed: " + delta);
+  }
+
+  private void makeRecordBody() {
+    startTiming("Allocating record");
+    record = new byte[(int)recordLen];
+    for (long i = 0; i < recordLen; i++) {
+      record[(int)i] = (byte) (i % 256);
+    }
+    stopTiming();
+  }
+
+  private void writeFile() throws Exception {
+    startTiming("Writing " + numRecords + " records to lob file");
+    LobFile.Writer w = LobFile.create(p, conf);
+    for (int i = 0; i < numRecords; i++) {
+      OutputStream out = w.writeBlobRecord(recordLen);
+      out.write(record);
+      out.close();
+      w.finishRecord();
+    }
+    w.close();
+    stopTiming();
+  }
+
+  private void readFile() throws Exception {
+    startTiming("Reading from lob file");
+    LobFile.Reader r = LobFile.open(p, conf);
+    int receivedRecs = 0;
+    byte [] inputBuffer = new byte[4096];
+    long recordSize = 0;
+    while (r.next()) {
+      receivedRecs++;
+      InputStream in = r.readBlobRecord();
+      while (true) {
+        int thisRead = in.read(inputBuffer);
+        if (-1 == thisRead) {
+          break;
+        }
+        recordSize += (long) thisRead;
+      }
+    }
+    r.close();
+    stopTiming();
+    System.out.println("Got " + receivedRecs + " records");
+    System.out.println("Read " + recordSize + " bytes");
+  }
+
+  private void run() throws Exception {
+    makeRecordBody();
+    writeFile();
+    readFile();
+  }
+
+  public static void main(String [] args) throws Exception {
+    LobFilePerfTest test = new LobFilePerfTest();
+    test.run();
+  }
+}
diff --git a/src/perftest/LobFileStressTest.java b/src/perftest/LobFileStressTest.java
new file mode 100644
index 00000000..bf9bf947
--- /dev/null
+++ b/src/perftest/LobFileStressTest.java
@@ -0,0 +1,391 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.*;
+import java.util.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.sqoop.io.*;
+
+/**
+ * Stress test LobFiles by writing a bunch of different files and reading
+ * them back in various ways.
+ * Run with: src/scripts/run-perftest.sh LobFileStressTest
+ */
+public class LobFileStressTest {
+
+  // Big records in testBigFile() are 5 GB each.
+  public final static long LARGE_RECORD_LEN = 5L * 1024L * 1024L * 1024L;
+
+  int numRandomTrials = 1000000;
+  Configuration conf;
+  boolean allPassed;
+
+  long lastCompressPos; // start offset of the last record in the file.
+  long lastRawPos;
+
+  public LobFileStressTest() {
+    conf = new Configuration();
+    conf.set("fs.default.name", "file:///");
+    allPassed = true;
+  }
+
+  private Path getPath(boolean compress) {
+    if (compress) {
+      return new Path("compressed.lob");
+    } else { 
+      return new Path("integers.lob");
+    }
+  }
+
+  private long getLastRecordPos(boolean compress) {
+    if (compress) {
+      return lastCompressPos;
+    } else {
+      return lastRawPos;
+    }
+  }
+
+  private void setLastRecordPos(long pos, boolean compress) {
+    if (compress) {
+      lastCompressPos = pos;
+    } else {
+      lastRawPos = pos;
+    }
+  }
+
+  private int getNumRecords(boolean compress) {
+    if (!compress) {
+      return 40000000; // 40 million.
+    } else {
+      return 5000000; // 5 million; the compressor is just too slow for 40M.
+    }
+  }
+  
+  private void writeIntegerFile(boolean compress) throws Exception {
+    boolean passed = false;
+    try {
+      System.out.print("Writing integers file. compress=" + compress + ". ");
+      Path p = getPath(compress);
+      FileSystem fs = FileSystem.getLocal(conf);
+      if (fs.exists(p)) {
+        fs.delete(p, false);
+      }
+      String codecName = compress ? "deflate" : null;
+      LobFile.Writer w = LobFile.create(p, conf, false, codecName);
+
+      int numRecords = getNumRecords(compress);
+      for (int i = 0; i < numRecords; i++) {
+        setLastRecordPos(w.tell(), compress);
+        OutputStream os = w.writeBlobRecord(0);
+        DataOutputStream dos = new DataOutputStream(os);
+        dos.writeInt(i);
+        dos.close();
+        os.close();
+      }
+
+      w.close();
+      System.out.println("PASS");
+      passed = true;
+    } finally {
+      if (!passed) {
+        allPassed = false;
+        System.out.println("FAIL");
+      }
+    }
+  }
+
+  private void testSequentialScan(boolean compress) throws Exception {
+    // Write a LobFile containing several million integers
+    // and read the entire resulting file, verify that they
+    // all appear in order.
+    boolean success = false;
+    System.out.print("Testing sequential scan. compress=" + compress + ". ");
+    try {
+      LobFile.Reader r = LobFile.open(getPath(compress), conf);
+      int numRecords = getNumRecords(compress);
+      for (int i = 0; i < numRecords; i++) {
+        r.next();
+        if (!r.isRecordAvailable()) {
+          throw new Exception("File ended early; i=" + i);
+        }
+        long id = r.getRecordId();
+        if (id != i) {
+          throw new Exception("record id mismatch; expected " + i
+              + " got " + id);
+        }
+
+        InputStream is = r.readBlobRecord();
+        DataInputStream dis = new DataInputStream(is);
+        int val = dis.readInt();
+        if (val != i) {
+          throw new Exception("Unexpected value in stream; expected " + i
+              + " got " + val);
+        }
+
+        try {
+          // Make sure we can't read additional data from the stream.
+          byte b = dis.readByte();
+          throw new Exception("Got unexpected extra byte : " + b);
+        } catch (EOFException eof) {
+          // Expected this. Ignore.
+        }
+
+        dis.close();
+      }
+      if (r.next()) {
+        // We should have finished reading everything already.
+        throw new Exception("Additional record was waiting at end of file");
+      }
+      r.close();
+      System.out.println("PASS");
+      success = true;
+    } finally {
+      if (!success) {
+        allPassed = false;
+        System.out.println("FAIL");
+      }
+    }
+  }
+
+  private void testSmallSeeks(boolean compress) throws Exception {
+    // Write a LobFile containing several million integers
+    // and seek to randomly selected records. Make sure we can
+    // access each one.
+    boolean success = false;
+    long seed = System.currentTimeMillis();
+    long lastRecordPos = getLastRecordPos(compress);
+    System.out.print("Testing random seeks. compress=" + compress + ". "
+        + ". seed=" + seed + ". lastPos=" + lastRecordPos +". ");
+    try {
+      LobFile.Reader r = LobFile.open(getPath(compress), conf);
+      Random rnd = new Random(seed);
+      long curRecord = -1; // current position starts before record 0.
+      int numRecords = getNumRecords(compress);
+      for (int i = 0; i < numRandomTrials; i++) {
+        if (rnd.nextInt(100) < 20 && curRecord != numRecords - 1) {
+          // Sequentially access the next record.
+          if (!r.next()) {
+            throw new Exception("cur record is " + curRecord
+                + " but sequential next() failed!");
+          }
+
+          curRecord++;
+        } else {
+          // Randomly find a record.
+          long targetPos = rnd.nextLong() % (lastRecordPos + 1);
+          r.seek(targetPos);
+          if (!r.next()) {
+            throw new Exception("Could not seek to record starting at "
+                + targetPos);
+          }
+
+          curRecord = r.getRecordId();
+          if (curRecord < 0 || curRecord >= numRecords) {
+            throw new Exception("Unexpected record id " + curRecord);
+          }
+        }
+
+        // In either case, read the record and verify the data's correct.
+        InputStream is = r.readBlobRecord();
+        DataInputStream dis = new DataInputStream(is);
+
+        int val = dis.readInt();
+        if (val != curRecord) {
+          throw new Exception("Unexpected value in stream; expected "
+              + curRecord + " got " + val);
+        }
+
+        try {
+          // Make sure we can't read additional data from the stream.
+          byte b = dis.readByte();
+          throw new Exception("Got unexpected extra byte : " + b);
+        } catch (EOFException eof) {
+          // Expected this. Ignore.
+        }
+
+        dis.close();
+      }
+
+      r.close();
+      System.out.println("PASS");
+      success = true;
+    } finally {
+      if (!success) {
+        allPassed = false;
+        System.out.println("FAIL");
+      }
+    }
+  }
+
+  private Path getBigFilePath(boolean compress) {
+    if (compress) {
+      return new Path("big-compressed.lob");
+    } else {
+      return new Path("big.lob");
+    }
+  }
+
+  /**
+   * Check that the bytes in buf are of the form 0 1 2 .. 254 255 0 1 2..
+   * but offset by the value in 'firstByte'. Returns the next byte value
+   * we would expect in an adjacent follow-up array of bytes.
+   */
+  private byte checkBuf(byte [] buf, int len, byte firstByte) throws Exception {
+    byte b = firstByte;
+    for (int i = 0; i < len; i++) {
+      if (buf[i] != b++) {
+        throw new Exception("Expected value " + b + " but got " + buf[i]);
+      }
+    }
+
+    return b;
+  }
+
+  /**
+   * Check that a big record (from testBigFile) matches its expected
+   * contract.
+   * The reader should be positioned at the beginning of the record
+   * with id 'expectedId'.
+   */
+  private void checkBigRecord(LobFile.Reader r, long expectedId)
+      throws Exception {
+    if (!r.isRecordAvailable()) {
+      throw new Exception("No record available");
+    }
+
+    if (r.getRecordId() != expectedId) {
+      throw new Exception("Expected record id=" + expectedId
+          + "; got " + r.getRecordId());
+    }
+
+    // Read all the data in this record, check that it is the data
+    // we expect. The bytes in each record increase 0 1 2 .. 254 255 0 1 2..
+    // but each record's start byte is offset by its record id.
+    long received = 0;
+    InputStream is = r.readBlobRecord();
+    long expected = LARGE_RECORD_LEN;
+    final int BUF_SIZE = 16384;
+    byte [] buf = new byte[BUF_SIZE];
+    byte last = (byte) r.getRecordId();
+    while (expected > 0) {
+      int thisRead = is.read(buf);
+      if (thisRead == -1) {
+        break;
+      }
+
+      last = checkBuf(buf, thisRead, last);
+      expected -= thisRead;
+    }
+
+    if (expected > 0) {
+      throw new Exception("Couldn't read all the data! expected "
+          + expected + " more bytes");
+    }
+    
+    if (is.read() != -1) {
+      throw new Exception("Got an extra byte! Expected no more data.");
+    }
+  }
+
+  private void testBigFile(boolean compress) throws Exception {
+    // Write a file containing 5 GB records.
+    
+    final int NUM_RECORDS = 5;
+    boolean passed = false;
+
+    try {
+      System.out.print("Testing large file operations. compress="
+          + compress + ". ");
+
+      Path p = getBigFilePath(compress);
+      long startOffsets [] = new long[NUM_RECORDS];
+
+      // Write the file. Five records, 5 GB a piece.
+      System.out.print("Testing write. ");
+      FileSystem fs = FileSystem.getLocal(conf);
+      if (fs.exists(p)) {
+        fs.delete(p, false);
+      }
+      String codecName = compress ? "deflate" : null;
+      System.out.println("record size: " + LARGE_RECORD_LEN);
+      LobFile.Writer w = LobFile.create(p, conf, false, codecName);
+      for (int i = 0; i < NUM_RECORDS; i++) {
+        startOffsets[i] = w.tell();
+        System.out.println("Starting record " + i + " at " + startOffsets[i]);
+        OutputStream os = w.writeBlobRecord(0);
+        for (long v = 0; v < LARGE_RECORD_LEN; v++) {
+          long byteVal = (((long) i) + v) & 0xFF;
+          os.write((int) byteVal);
+        }
+        os.close();
+      }
+      w.close();
+      System.out.println("PASS");
+
+      // Iterate past three records, read the fourth.
+      System.out.print("Testing iterated skipping. ");
+      LobFile.Reader r = LobFile.open(p, conf);
+      for (int i = 0; i < 4; i++) {
+        r.next();
+      }
+
+      checkBigRecord(r, 3);
+      System.out.println("PASS");
+
+      // Seek directly to record 2, read it through.
+      System.out.print("Testing large backward seek. ");
+      r.seek(startOffsets[2]);
+      r.next();
+      checkBigRecord(r, 2);
+      System.out.println("PASS");
+
+      passed = true;
+    } finally {
+      if (!passed) {
+        allPassed = false;
+        System.out.println("FAIL");
+      }
+    }
+  }
+
+
+  private void run() throws Exception {
+    writeIntegerFile(true);
+    writeIntegerFile(false);
+    testSequentialScan(false);
+    testSmallSeeks(false);
+    testSequentialScan(true);
+    testSmallSeeks(true);
+    testBigFile(false);
+    testBigFile(true);
+
+    if (allPassed) {
+      System.out.println("Tests passed.");
+    } else {
+      System.out.println("All tests did not pass!");
+      System.exit(1);
+    }
+
+  }
+
+  public static void main(String [] args) throws Exception {
+    LobFileStressTest test = new LobFileStressTest();
+    test.run();
+  }
+}
diff --git a/src/perftest/README-perftest.txt b/src/perftest/README-perftest.txt
new file mode 100644
index 00000000..ec782974
--- /dev/null
+++ b/src/perftest/README-perftest.txt
@@ -0,0 +1,40 @@
+
+   Licensed to Cloudera, Inc. under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+
+= Performance and Stress Tests
+
+The files in this directory represent performance or stress tests of aspects
+of Sqoop. These are intended to be run by a developer as a part of a lengthy
+QA process but are not the "primary" tests which are expected to be run for
+every build.
+
+
+== Compiling
+
+To compile the tests in this directory, run 'ant jar compile-perf-test' in the
+project root.
+
+== Running
+
+After compiling the performance tests, you can run them with:
+
+$ src/scripts/run-perftest.sh <PerfTestClassName> [<args...>]
+
+e.g.:
+
+$ src/scripts/run-perftest.sh LobFileStressTest
+
diff --git a/src/scripts/run-perftest.sh b/src/scripts/run-perftest.sh
new file mode 100755
index 00000000..6e459ca2
--- /dev/null
+++ b/src/scripts/run-perftest.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+#   Licensed to Cloudera, Inc. under one or more
+#   contributor license agreements.  See the NOTICE file distributed with
+#   this work for additional information regarding copyright ownership.
+#   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+#   (the "License"); you may not use this file except in compliance with
+#   the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+
+# run-perftest.sh
+# USAGE:
+# ./run-perftest.sh PerfTestClassName [arg arg arg...]
+#
+# This script will run one of the classes from src/perftest/ with the
+# correct classpath set up.
+
+bin=`dirname $0`
+bin=`cd ${bin} && pwd`
+
+# This is run in src/scripts/
+SQOOP_HOME="${bin}/../../"
+
+# Set up environment and classpath
+source ${SQOOP_HOME}/bin/configure-sqoop
+
+PERFTEST_CLASSES=${SQOOP_HOME}/build/perftest/classes
+
+export HADOOP_CLASSPATH=${PERFTEST_CLASSES}:${SQOOP_JAR}:${HADOOP_CLASSPATH}
+${HADOOP_HOME}/bin/hadoop "$@"
+
diff --git a/src/test/org/apache/hadoop/sqoop/SmokeTests.java b/src/test/org/apache/hadoop/sqoop/SmokeTests.java
index 6aadf3c2..cb0c766b 100644
--- a/src/test/org/apache/hadoop/sqoop/SmokeTests.java
+++ b/src/test/org/apache/hadoop/sqoop/SmokeTests.java
@@ -20,6 +20,7 @@
 
 import org.apache.hadoop.sqoop.hive.TestHiveImport;
 import org.apache.hadoop.sqoop.hive.TestTableDefWriter;
+import org.apache.hadoop.sqoop.io.TestLobFile;
 import org.apache.hadoop.sqoop.io.TestSplittableBufferedWriter;
 import org.apache.hadoop.sqoop.lib.TestFieldFormatter;
 import org.apache.hadoop.sqoop.lib.TestRecordParser;
@@ -68,6 +69,7 @@ public static Test suite() {
     suite.addTestSuite(TestClobRef.class);
     suite.addTestSuite(TestLargeObjectLoader.class);
     suite.addTestSuite(TestDirectImportUtils.class);
+    suite.addTestSuite(TestLobFile.class);
     suite.addTest(MapreduceTests.suite());
 
     return suite;
diff --git a/src/test/org/apache/hadoop/sqoop/io/TestLobFile.java b/src/test/org/apache/hadoop/sqoop/io/TestLobFile.java
new file mode 100644
index 00000000..3d53a6b2
--- /dev/null
+++ b/src/test/org/apache/hadoop/sqoop/io/TestLobFile.java
@@ -0,0 +1,577 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop.io;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Reader;
+import java.io.Writer;
+import java.nio.CharBuffer;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Test the LobFile reader/writer implementation.
+ */
+public class TestLobFile extends TestCase {
+
+  public static final Log LOG = LogFactory.getLog(
+        TestLobFile.class.getName());
+
+  public static final Path TEMP_BASE_DIR;
+
+  static {
+    String tmpDir = System.getProperty("test.build.data", "/tmp/");
+    if (!tmpDir.endsWith(File.separator)) {
+      tmpDir = tmpDir + File.separator;
+    }
+
+    TEMP_BASE_DIR = new Path(new Path(tmpDir), "lobtest");
+  }
+
+  private Configuration conf;
+  private FileSystem fs;
+
+  public void setUp() throws Exception {
+    conf = new Configuration();
+    conf.set("fs.default.name", "file:///");
+
+    fs = FileSystem.getLocal(conf);
+    fs.mkdirs(TEMP_BASE_DIR);
+  }
+
+  private long [] writeClobFile(Path p, String codec,
+      String... records) throws Exception {
+    if (fs.exists(p)) {
+      fs.delete(p, false);
+    }
+
+    // memorize the offsets of each record we write.
+    long [] offsets = new long[records.length];
+
+    // Create files with four entries per index segment.
+    LobFile.Writer writer = LobFile.create(p, conf, true, codec, 4);
+
+    int i = 0;
+    for (String r : records) {
+      offsets[i++] = writer.tell();
+      Writer w = writer.writeClobRecord(r.length());
+      w.write(r);
+      w.close();
+    }
+
+    writer.close();
+    return offsets;
+  }
+
+  private void verifyClobFile(Path p, String... expectedRecords) throws Exception {
+    
+    LobFile.Reader reader = LobFile.open(p, conf);
+
+    int recNum = 0;
+
+    while (reader.next()) {
+      // We should have a record of the same length as the expected one.
+      String expected = expectedRecords[recNum];
+      assertTrue(reader.isRecordAvailable());
+      assertEquals(expected.length(), reader.getRecordLen());
+      Reader r = reader.readClobRecord();
+
+      // Read in the record and assert that we got enough characters out.
+      CharBuffer buf = CharBuffer.allocate(expected.length());
+      int bytesRead = 0;
+      while (bytesRead < expected.length()) {
+        int thisRead = r.read(buf);
+        LOG.info("performed read of " + thisRead + " chars");
+        if (-1 == thisRead) {
+          break;
+        }
+
+        bytesRead += thisRead;
+      }
+
+      LOG.info("Got record of " + bytesRead + " chars");
+      assertEquals(expected.length(), bytesRead);
+      char [] charData = buf.array();
+      String finalRecord = new String(charData);
+      assertEquals(expected, finalRecord);
+
+      recNum++;
+    }
+
+    // Check that we got everything.
+    assertEquals(expectedRecords.length, recNum);
+
+    reader.close();
+
+    try {
+      reader.next();
+      fail("Expected IOException calling next after close");
+    } catch (IOException ioe) {
+      // expected this.
+    }
+
+    // A second close shouldn't hurt anything. This should be a no-op.
+    reader.close();
+  }
+
+  private void runClobFileTest(Path p, String codec,
+      String... records) throws Exception {
+    writeClobFile(p, codec, records);
+    verifyClobFile(p, records);
+    fs.delete(p, false);
+  }
+
+  public void testEmptyRecord() throws Exception {
+    runClobFileTest(new Path(TEMP_BASE_DIR, "empty.lob"), null);
+  }
+
+  public void testSingleRecord() throws Exception {
+    runClobFileTest(new Path(TEMP_BASE_DIR, "single.lob"),
+        null, "this is a single record!");
+  }
+
+  public void testMultiRecords() throws Exception {
+    runClobFileTest(new Path(TEMP_BASE_DIR, "multi.lob"),
+        CodecMap.NONE,
+        "this is the first record",
+        "this is the second record. I assure you that this record is long.",
+        "yet one more record graces this file.");
+  }
+
+  public void testMultiIndexSegments() throws Exception {
+    // Test that we can use multiple IndexSegments.
+    runClobFileTest(new Path(TEMP_BASE_DIR, "multi-index.lob"),
+        null,
+        "this is the first record",
+        "this is the second record. I assure you that this record is long.",
+        "record number three",
+        "last one in first index segment",
+        "first in the second index segment",
+        "yet one more record graces this file.");
+  }
+
+  /**
+   * Run a test where we read only a fraction of the first record,
+   * but then read the second record completely. Verify that we
+   * can re-align on a record boundary correctly. This test requires
+   * at least 3 records.
+   * @param p the path to the file to create.
+   * @param firstLine the first line of the first reord
+   * @param records All of the records to write to the file.
+   */
+  private void runLineAndRecordTest(Path p, String firstLine,
+      String... records) throws Exception {
+
+    assertTrue("This test requires 3+ records", records.length > 2);
+
+    writeClobFile(p, null, records);
+
+    LobFile.Reader reader = LobFile.open(p, conf);
+
+    // We should not yet be aligned.
+    assertFalse(reader.isRecordAvailable());
+    assertTrue(reader.next());
+    // Now we should be.
+    assertTrue(reader.isRecordAvailable());
+
+    // Read just one line from the record.
+    Reader r = reader.readClobRecord();
+    BufferedReader br = new BufferedReader(r);
+    String line = br.readLine();
+    assertEquals(firstLine, line);
+
+    br.close();
+    r.close();
+
+    // We should no longer be aligned on a record start.
+    assertFalse(reader.isRecordAvailable());
+
+    // We should now be able to get to record two.
+    assertTrue(reader.next());
+
+    // This should be nicely aligned even if the first record was not
+    // completely consumed by a client.
+    r = reader.readClobRecord();
+    CharBuffer buf = CharBuffer.allocate(records[1].length());
+    r.read(buf);
+    r.close();
+    char [] chars = buf.array();
+    String s = new String(chars);
+    assertEquals(records[1], s);
+
+    // Close the reader before we consume the entire file. 
+    reader.close();
+    assertFalse(reader.isRecordAvailable());
+  }
+
+  public void testVeryShortRead() throws Exception {
+    // Read only a small fraction of a record, ensure that we can
+    // read the next record, even when we've left more than a 16-byte
+    // quantity in the readahead buffer.
+
+    Path p = new Path(TEMP_BASE_DIR, "shortread.lob");
+    final String firstLine = "line1";
+    final String secondLine =
+        "This contains much more in the record than just one line.";
+    final String record2 = "here is the second record.";
+    final String record3 = "The 3rd record, which we won't actually read.";
+
+    runLineAndRecordTest(p, firstLine,
+        firstLine + "\n" + secondLine,
+        record2,
+        record3);
+
+  }
+
+  public void testIncompleteOverread() throws Exception {
+    // Read most of the first record so that we partially consume the
+    // next record start mark; make sure we realign properly.
+
+    Path p = new Path(TEMP_BASE_DIR, "longread.lob");
+    final String firstLine = "this is a really long line of text to read!";
+    final String secondLine = "not this.";
+    final String record2 = "Here is yet another record to verify.";
+    final String record3 = "Nobody cares about record 3.";
+
+    runLineAndRecordTest(p, firstLine,
+        firstLine + "\n" + secondLine,
+        record2,
+        record3);
+  }
+
+  public void testSeekToRecord() throws Exception {
+    // Seek past the first two records and read the third.
+
+    Path p = new Path(TEMP_BASE_DIR, "seek.lob");
+    String [] records = {
+      "this is the first record!",
+      "here comes record number two. It is a bit longer.",
+      "this is the third record. we can read it."
+    };
+
+    // Write the file and memorize when the third record starts.
+    LobFile.Writer writer = LobFile.create(p, conf, true);
+
+    int recNum = 0;
+    long rec3Start = 0;
+    for (String r : records) {
+      Writer w = writer.writeClobRecord(r.length());
+      w.write(r);
+      w.close();
+      writer.finishRecord();
+      if (recNum == 1) {
+        rec3Start = writer.tell();
+        LOG.info("Record three start: " + rec3Start);
+      }
+      recNum++;
+    }
+
+    writer.close();
+
+    // Now reopen the file for read, seek to the third record, and get it.
+    LobFile.Reader reader = LobFile.open(p, conf);
+    reader.seek(rec3Start);
+    assertTrue(reader.next());
+    assertTrue(reader.isRecordAvailable());
+    assertEquals(rec3Start, reader.getRecordOffset());
+
+    Reader r = reader.readClobRecord();
+    CharBuffer buf = CharBuffer.allocate(records[2].length());
+    r.read(buf);
+    r.close();
+    char [] chars = buf.array();
+    String s = new String(chars);
+    assertEquals(records[2], s);
+    
+    r.close();
+    reader.close();
+  }
+
+
+  /** Verifies that the next record in the LobFile is the expected one. */
+  private void verifyNextRecord(LobFile.Reader reader, long expectedId,
+      String expectedRecord) throws Exception {
+    assertTrue(reader.next());
+    assertTrue(reader.isRecordAvailable());
+    assertEquals(expectedId, reader.getRecordId());
+
+    Reader r = reader.readClobRecord();
+    CharBuffer buf = CharBuffer.allocate(expectedRecord.length());
+    int bytesRead = 0;
+    while (bytesRead < expectedRecord.length()) {
+      int thisRead = r.read(buf);
+      if (-1 == thisRead) {
+        break;
+      }
+
+      bytesRead += thisRead;
+    }
+
+    LOG.info("Got record of " + bytesRead + " chars");
+    assertEquals(expectedRecord.length(), bytesRead);
+
+    char [] charData = buf.array();
+    String finalRecord = new String(charData);
+    assertEquals(expectedRecord, finalRecord);
+  }
+
+  public void testManySeeks() throws Exception {
+    // Test that we can do gymnastics with seeking between records.
+
+    Path p = new Path(TEMP_BASE_DIR, "manyseeks.lob");
+
+    String [] records = {
+      "first record",
+      "second record",
+      "the third record",
+      "rec4 is the last in IndexSeg 0",
+      "rec5 is first in IndexSeg 1",
+      "rec6 is yet another record",
+      "rec7 is starting to feel boring",
+      "rec8 is at the end of seg 1",
+      "rec9 is all by itself in seg 2"
+    };
+
+    // Write the records to a file, save their offsets.
+    long [] offsets = writeClobFile(p, null, records);
+
+    // Sanity check that we can stream the file.
+    verifyClobFile(p, records);
+
+    // Open a handle to the file.
+    LobFile.Reader reader = LobFile.open(p, conf);
+
+    // Seeking to offset 0 should return the first record.
+    reader.seek(0);
+    verifyNextRecord(reader, 0, records[0]);
+
+    // Seek to the last item in the first IndexSegment.
+    reader.seek(offsets[3]);
+    verifyNextRecord(reader, 3, records[3]);
+
+    // Seek to just ahead of that same record.
+    reader.seek(offsets[3] - 10);
+    verifyNextRecord(reader, 3, records[3]);
+
+    // Seek (backwards) to the first record.
+    reader.seek(offsets[0]);
+    verifyNextRecord(reader, 0, records[0]);
+
+    // Seek to first record in second IndexSegment.
+    reader.seek(offsets[4]);
+    verifyNextRecord(reader, 4, records[4]);
+
+    // Move backwards.
+    reader.seek(0);
+
+    // Seek to "no man's land" between last offset in first IndexSeg
+    // and the first offset in second IndexSegment. Result should be
+    // the first record in second InexSegment.
+    reader.seek(offsets[4] - 10);
+    verifyNextRecord(reader, 4, records[4]);
+
+    // Seek to past the last record. No record should be returned.
+    reader.seek(offsets[8] + 4);
+    assertFalse("Found a record past last record start.", reader.next());
+
+    // Seek to somewhere in the middle of IndexSegment 0.
+    // This should recover just fine.
+    reader.seek(offsets[2]);
+    verifyNextRecord(reader, 2, records[2]);
+
+    // Seek to last record in IndexSegment 1.
+    reader.seek(offsets[3] - 1);
+    verifyNextRecord(reader, 3, records[3]);
+
+    // And make sure that iteration picks up naturally from there.
+    verifyNextRecord(reader, 4, records[4]);
+
+    // Seek well past the end of the file. No record should be returned.
+    reader.seek(50000000);
+    assertFalse("Found a record past expected end-of-file", reader.next());
+
+    // Seek to somewhere in the index.
+    reader.seek(offsets[8] + 32);
+    assertFalse("Found a record past beginning of index", reader.next());
+
+    // Seek to the last record (exact hit). This is a singleton IndexSegment.
+    reader.seek(offsets[8]);
+    verifyNextRecord(reader, 8, records[8]);
+
+    // Seek to no-man's-land ahead of last record.
+    reader.seek(offsets[8] - 3);
+    verifyNextRecord(reader, 8, records[8]);
+
+    reader.close();
+  }
+
+  /**
+   * Verifies that a record to be read from a lob file has
+   * as many bytes as we expect, and that the bytes are what we
+   * expect them to be. Assumes that the bytes are such that
+   * input[i] == i + offset.
+   * @param reader the LobFile reader to consume data from
+   * @param expectedDeclaredLen the size we expect the LobFile to declare
+   * its record length as.
+   * @param expectedActualLen the true number of bytes we expect to read in
+   * the record.
+   * @param offset the offset amount for each of the elements of the array.
+   */
+  private void verifyBlobRecord(LobFile.Reader reader,
+      long expectedDeclaredLen, long expectedActualLen,
+      int offset) throws Exception {
+
+    assertTrue(reader.next());
+    assertTrue(reader.isRecordAvailable());
+    assertEquals(expectedDeclaredLen, reader.getRecordLen());
+
+    InputStream is = reader.readBlobRecord();
+
+    byte [] bytes = new byte[(int) expectedActualLen];
+    int numRead = is.read(bytes);
+    assertEquals(expectedActualLen, numRead);
+
+    for (int i = 0; i < numRead; i++) {
+      assertEquals(i + offset, (int) bytes[i]);
+    }
+
+    is.close();
+  }
+
+  /**
+   * Write a binary record to a LobFile. This allows the declared length
+   * of the record to disagree with the actual length (the actual length
+   * should be &gt;= the declared length).
+   * The record written will have values v[i] = i + offset.
+   * @param writer the LobFile writer to put the record into
+   * @param declaredLen the length value written into the file itself
+   * @param actualLen the true number of bytes to write
+   * @param offset an amount to adjust each record's byte values by.
+   */
+  private void writeBlobRecord(LobFile.Writer writer, long declaredLen,
+      long actualLen, int offset) throws Exception {
+    OutputStream os = writer.writeBlobRecord(declaredLen);
+    for (int i = 0; i < actualLen; i++) {
+      os.write(i + offset);
+    }
+
+    os.close();
+    writer.finishRecord();
+  }
+
+  /**
+   * Verifies a number of records that all have the same declared
+   * and actual record lengths.
+   * @param p the path to the LobFile to open
+   * @param numRecords the number of records to expect
+   * @param declaredLen the declared length of each record in the file
+   * @param actualLen the true number of bytes we expect to read per record.
+   */
+  private void verifyBlobRecords(Path p, int numRecords,
+      long declaredLen, long actualLen) throws Exception {
+
+    LobFile.Reader reader = LobFile.open(p, conf);
+    for (int i = 0; i < numRecords; i++) {
+      verifyBlobRecord(reader, declaredLen, actualLen, i);
+    }
+    assertFalse(reader.next());
+    reader.close();
+  }
+
+  public void testBinaryRecords() throws Exception {
+    // Write a BLOB file and read it all back.
+
+    final long RECORD_LEN = 32;
+    final int NUM_RECORDS = 2;
+    Path p = new Path(TEMP_BASE_DIR, "binary.lob");
+    LobFile.Writer writer = LobFile.create(p, conf);
+
+    for (int i = 0; i < NUM_RECORDS; i++) {
+      writeBlobRecord(writer, RECORD_LEN, RECORD_LEN, i);
+    }
+
+    writer.close();
+
+    // Now check the read-back on those records.
+    verifyBlobRecords(p, NUM_RECORDS, RECORD_LEN, RECORD_LEN);
+  }
+
+  public void testOverLengthBinaryRecord() throws Exception {
+    // Write a record with a declared length shorter than the
+    // actual length, and read it back.
+
+    final long ACTUAL_RECORD_LEN = 48;
+    final long DECLARED_RECORD_LEN = 32;
+    final int NUM_RECORDS = 2;
+
+    Path p = new Path(TEMP_BASE_DIR, "overlength.lob");
+    LobFile.Writer writer = LobFile.create(p, conf);
+
+    for (int i = 0; i < NUM_RECORDS; i++) {
+      writeBlobRecord(writer, DECLARED_RECORD_LEN, ACTUAL_RECORD_LEN, i);
+    }
+
+    writer.close();
+
+    // Now read them back.
+    verifyBlobRecords(p, NUM_RECORDS, DECLARED_RECORD_LEN, ACTUAL_RECORD_LEN);
+  }
+
+  private void runCompressedTest(String codec) throws Exception {
+    LOG.info("Testing with codec: " + codec);
+    Path p = new Path(TEMP_BASE_DIR, "compressed-" + codec + ".lob");
+    String [] records = {
+       "this is the first record, It should be compressed a lot!",
+       "record 2 record 2 record 2 record 2 2 2 2 2 2 2 2 2 2 2 2",
+       "and a third and a third yes this is the third"
+    };
+
+    runClobFileTest(p, codec, records);
+  }
+
+  public void testCompressedFile() throws Exception {
+    // Test all the various compression codecs.
+
+    // The following values for 'codec' should pass.
+    runCompressedTest(null);
+    runCompressedTest(CodecMap.NONE);
+    runCompressedTest(CodecMap.DEFLATE);
+
+    try {
+      // We expect this to throw UnsupportedCodecException
+      // because this class is not included in our package.
+      runCompressedTest(CodecMap.LZO);
+      fail("Expected unsupported codec exception for lzo");
+    } catch (UnsupportedCodecException uce) {
+      // We pass.
+      LOG.info("Got unsupported codec exception for lzo; expected -- good.");
+    }
+  }
+}
+
diff --git a/src/test/org/apache/hadoop/sqoop/lib/TestBlobRef.java b/src/test/org/apache/hadoop/sqoop/lib/TestBlobRef.java
index fd220912..d76746f9 100644
--- a/src/test/org/apache/hadoop/sqoop/lib/TestBlobRef.java
+++ b/src/test/org/apache/hadoop/sqoop/lib/TestBlobRef.java
@@ -27,7 +27,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-
+import org.apache.hadoop.sqoop.io.LobFile;
 
 /**
  * Test that the BlobRef.parse() method does the right thing.
@@ -47,13 +47,25 @@ public void testInline() throws IOException {
   }
 
   public void testEmptyFile() {
-    BlobRef r = BlobRef.parse("externalBlob()");
+    BlobRef r = BlobRef.parse("externalLob()");
+    assertFalse(r.isExternal());
+
+    r = BlobRef.parse("externalLob(lf,,0,0)");
     assertTrue(r.isExternal());
-    assertEquals("externalBlob()", r.toString());
+    assertEquals("externalLob(lf,,0,0)", r.toString());
   }
 
   public void testInlineNearMatch() {
-    BlobRef r = BlobRef.parse("externalBlob(foo)bar");
+    BlobRef r = BlobRef.parse("externalLob(foo)bar");
+    assertFalse(r.isExternal());
+
+    r = BlobRef.parse("externalLob(foo)");
+    assertFalse(r.isExternal());
+
+    r = BlobRef.parse("externalLob(lf,foo)");
+    assertFalse(r.isExternal());
+
+    r = BlobRef.parse("externalLob(lf,foo,1,2)x");
     assertFalse(r.isExternal());
   }
 
@@ -89,7 +101,6 @@ private void doExternalTest(final byte [] DATA, final String FILENAME)
     String tmpDir = System.getProperty("test.build.data", "/tmp/");
 
     Path tmpPath = new Path(tmpDir);
-
     Path blobFile = new Path(tmpPath, FILENAME);
 
     // make any necessary parent dirs.
@@ -98,14 +109,20 @@ private void doExternalTest(final byte [] DATA, final String FILENAME)
       fs.mkdirs(blobParent);
     }
 
-    OutputStream os = fs.create(blobFile);
+    LobFile.Writer lw = LobFile.create(blobFile, conf, false);
     try {
+      long off = lw.tell();
+      long len = DATA.length;
+      OutputStream os = lw.writeBlobRecord(len);
       os.write(DATA, 0, DATA.length);
       os.close();
+      lw.close();
 
-      BlobRef blob = BlobRef.parse("externalBlob(" + FILENAME + ")");
+      String refString = "externalLob(lf," + FILENAME
+          + "," + off + "," + len + ")";
+      BlobRef blob = BlobRef.parse(refString);
       assertTrue(blob.isExternal());
-      assertEquals("externalBlob(" + FILENAME + ")", blob.toString());
+      assertEquals(refString, blob.toString());
       InputStream is = blob.getDataStream(conf, tmpPath);
       assertNotNull(is);
 
diff --git a/src/test/org/apache/hadoop/sqoop/lib/TestClobRef.java b/src/test/org/apache/hadoop/sqoop/lib/TestClobRef.java
index a2dcbab9..241452b9 100644
--- a/src/test/org/apache/hadoop/sqoop/lib/TestClobRef.java
+++ b/src/test/org/apache/hadoop/sqoop/lib/TestClobRef.java
@@ -27,7 +27,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-
+import org.apache.hadoop.sqoop.io.LobFile;
 
 /**
  * Test parsing of ClobRef objects.
@@ -45,7 +45,7 @@ public void testInline() throws IOException {
     assertFalse(r.isExternal());
     assertEquals("foo", r.toString());
 
-    Reader reader = r.getDataReader(null, null);
+    Reader reader = r.getDataStream(null, null);
     assertNotNull(reader);
     char [] buf = new char[4096];
     int chars = reader.read(buf, 0, 4096);
@@ -56,15 +56,31 @@ public void testInline() throws IOException {
   }
 
   public void testEmptyFile() {
-    ClobRef r = ClobRef.parse("externalClob()");
+    ClobRef r = ClobRef.parse("externalLob()");
+    assertFalse(r.isExternal());
+    assertEquals("externalLob()", r.toString());
+
+    r = ClobRef.parse("externalLob(lf,,0,0)");
     assertTrue(r.isExternal());
-    assertEquals("externalClob()", r.toString());
+    assertEquals("externalLob(lf,,0,0)", r.toString());
   }
 
   public void testInlineNearMatch() {
-    ClobRef r = ClobRef.parse("externalClob(foo)bar");
+    ClobRef r = ClobRef.parse("externalLob(foo)bar");
     assertFalse(r.isExternal());
-    assertEquals("externalClob(foo)bar", r.toString());
+    assertEquals("externalLob(foo)bar", r.toString());
+
+    r = ClobRef.parse("externalLob(foo)");
+    assertFalse(r.isExternal());
+    assertEquals("externalLob(foo)", r.getData());
+
+    r = ClobRef.parse("externalLob(lf,foo)");
+    assertFalse(r.isExternal());
+    assertEquals("externalLob(lf,foo)", r.getData());
+
+    r = ClobRef.parse("externalLob(lf,foo,1,2)x");
+    assertFalse(r.isExternal());
+    assertEquals("externalLob(lf,foo,1,2)x", r.getData());
   }
 
   public void testExternal() throws IOException {
@@ -99,7 +115,6 @@ private void doExternalTest(final String DATA, final String FILENAME)
     String tmpDir = System.getProperty("test.build.data", "/tmp/");
 
     Path tmpPath = new Path(tmpDir);
-
     Path clobFile = new Path(tmpPath, FILENAME);
 
     // make any necessary parent dirs.
@@ -108,16 +123,21 @@ private void doExternalTest(final String DATA, final String FILENAME)
       fs.mkdirs(clobParent);
     }
 
-    BufferedWriter w = new BufferedWriter(new OutputStreamWriter(
-        fs.create(clobFile)));
+    LobFile.Writer lw = LobFile.create(clobFile, conf, true);
     try {
+      long off = lw.tell();
+      long len = DATA.length();
+      Writer w = lw.writeClobRecord(len);
       w.append(DATA);
       w.close();
+      lw.close();
 
-      ClobRef clob = ClobRef.parse("externalClob(" + FILENAME + ")");
+      String refString = "externalLob(lf," + FILENAME 
+                + "," + off + "," + len + ")";
+      ClobRef clob = ClobRef.parse(refString);
       assertTrue(clob.isExternal());
-      assertEquals("externalClob(" + FILENAME + ")", clob.toString());
-      Reader r = clob.getDataReader(conf, tmpPath);
+      assertEquals(refString, clob.toString());
+      Reader r = clob.getDataStream(conf, tmpPath);
       assertNotNull(r);
 
       char [] buf = new char[4096];
diff --git a/src/test/org/apache/hadoop/sqoop/lib/TestLargeObjectLoader.java b/src/test/org/apache/hadoop/sqoop/lib/TestLargeObjectLoader.java
index 30b92853..204af882 100644
--- a/src/test/org/apache/hadoop/sqoop/lib/TestLargeObjectLoader.java
+++ b/src/test/org/apache/hadoop/sqoop/lib/TestLargeObjectLoader.java
@@ -87,8 +87,9 @@ public void testReadClobRef()
     clob = loader.readClobRef(0, resultSet);
     assertNotNull(clob);
     assertTrue(clob.isExternal());
+    loader.close();
     mapContext.getOutputCommitter().commitTask(mapContext);
-    Reader r = clob.getDataReader(conf, outDir);
+    Reader r = clob.getDataStream(conf, outDir);
     char [] buf = new char[4096];
     int chars = r.read(buf, 0, 4096);
     r.close();
@@ -115,6 +116,7 @@ public void testReadBlobRef()
     blob = loader.readBlobRef(0, resultSet);
     assertNotNull(blob);
     assertTrue(blob.isExternal());
+    loader.close();
     mapContext.getOutputCommitter().commitTask(mapContext);
     InputStream is = blob.getDataStream(conf, outDir);
     byte [] buf = new byte[4096];