/*
 * Copyright (C) 2008-2009, Google Inc.
 * Copyright (C) 2007, Robin Rosenberg <robin.rosenberg@dewire.com>
 * Copyright (C) 2006-2008, Shawn O. Pearce <spearce@spearce.org>
 * and other copyright owners as documented in the project's IP log.
 *
 * This program and the accompanying materials are made available
 * under the terms of the Eclipse Distribution License v1.0 which
 * accompanies this distribution, is reproduced below, and is
 * available at http://www.eclipse.org/org/documents/edl-v10.php
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or
 * without modification, are permitted provided that the following
 * conditions are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above
 *   copyright notice, this list of conditions and the following
 *   disclaimer in the documentation and/or other materials provided
 *   with the distribution.
 *
 * - Neither the name of the Eclipse Foundation, Inc. nor the
 *   names of its contributors may be used to endorse or promote
 *   products derived from this software without specific prior
 *   written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package org.eclipse.jgit.internal.storage.file;

import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP;

import java.io.EOFException;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.AccessDeniedException;
import java.nio.file.NoSuchFileException;
import java.text.MessageFormat;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.CRC32;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;

import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.errors.LargeObjectException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.errors.NoPackSignatureException;
import org.eclipse.jgit.errors.PackInvalidException;
import org.eclipse.jgit.errors.PackMismatchException;
import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
import org.eclipse.jgit.errors.UnpackException;
import org.eclipse.jgit.errors.UnsupportedPackIndexVersionException;
import org.eclipse.jgit.errors.UnsupportedPackVersionException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
import org.eclipse.jgit.internal.storage.pack.PackExt;
import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
import org.eclipse.jgit.lib.AbbreviatedObjectId;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.util.LongList;
import org.eclipse.jgit.util.NB;
import org.eclipse.jgit.util.RawParseUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

A Git version 2 pack file representation. A pack file contains Git objects in
delta packed format yielding high compression of lots of object where some
objects are similar.
/**
 * A Git version 2 pack file representation. A pack file contains Git objects in
 * delta packed format yielding high compression of lots of object where some
 * objects are similar.
 */
public class PackFile implements Iterable<PackIndex.MutableEntry> {
	private final static Logger LOG = LoggerFactory.getLogger(PackFile.class);
	
Sorts PackFiles to be most recently created to least recently created. /** Sorts PackFiles to be most recently created to least recently created. */
	public static final Comparator<PackFile> SORT = new Comparator<PackFile>() {
		@Override
		public int compare(PackFile a, PackFile b) {
			return b.packLastModified.compareTo(a.packLastModified);
		}
	};

	private final File packFile;

	private final int extensions;

	private File keepFile;

	private volatile String packName;

	final int hash;

	private RandomAccessFile fd;

	
Serializes reads performed against fd. /** Serializes reads performed against {@link #fd}. */
	private final Object readLock = new Object();

	long length;

	private int activeWindows;

	private int activeCopyRawData;

	Instant packLastModified;

	private PackFileSnapshot fileSnapshot;

	private volatile boolean invalid;

	private volatile Exception invalidatingCause;

	private boolean invalidBitmap;

	private AtomicInteger transientErrorCount = new AtomicInteger();

	private byte[] packChecksum;

	private volatile PackIndex loadedIdx;

	private PackReverseIndex reverseIdx;

	private PackBitmapIndex bitmapIdx;

	
Objects we have tried to read, and discovered to be corrupt.

The list is allocated after the first corruption is found, and filled in
as more entries are discovered. Typically this list is never used, as
pack files do not usually contain corrupt objects.
/**
	 * Objects we have tried to read, and discovered to be corrupt.
	 * <p>
	 * The list is allocated after the first corruption is found, and filled in
	 * as more entries are discovered. Typically this list is never used, as
	 * pack files do not usually contain corrupt objects.
	 */
	private volatile LongList corruptObjects;

	
Construct a reader for an existing, pre-indexed packfile.
Params:
packFile – 
           path of the .pack file holding the data.
extensions – 
           additional pack file extensions with the same base as the pack/**
	 * Construct a reader for an existing, pre-indexed packfile.
	 *
	 * @param packFile
	 *            path of the <code>.pack</code> file holding the data.
	 * @param extensions
	 *            additional pack file extensions with the same base as the pack
	 */
	public PackFile(File packFile, int extensions) {
		this.packFile = packFile;
		this.fileSnapshot = PackFileSnapshot.save(packFile);
		this.packLastModified = fileSnapshot.lastModifiedInstant();
		this.extensions = extensions;

		// Multiply by 31 here so we can more directly combine with another
		// value in WindowCache.hash(), without doing the multiply there.
		//
		hash = System.identityHashCode(this) * 31;
		length = Long.MAX_VALUE;
	}

	private PackIndex idx() throws IOException {
		PackIndex idx = loadedIdx;
		if (idx == null) {
			synchronized (this) {
				idx = loadedIdx;
				if (idx == null) {
					if (invalid) {
						throw new PackInvalidException(packFile, invalidatingCause);
					}
					try {
						long start = System.currentTimeMillis();
						idx = PackIndex.open(extFile(INDEX));
						if (LOG.isDebugEnabled()) {
							LOG.debug(String.format(
									"Opening pack index %s, size %.3f MB took %d ms", //$NON-NLS-1$
									extFile(INDEX).getAbsolutePath(),
									Float.valueOf(extFile(INDEX).length()
											/ (1024f * 1024)),
									Long.valueOf(System.currentTimeMillis()
											- start)));
						}

						if (packChecksum == null) {
							packChecksum = idx.packChecksum;
							fileSnapshot.setChecksum(
									ObjectId.fromRaw(packChecksum));
						} else if (!Arrays.equals(packChecksum,
								idx.packChecksum)) {
							throw new PackMismatchException(MessageFormat
									.format(JGitText.get().packChecksumMismatch,
											packFile.getPath(),
											ObjectId.fromRaw(packChecksum)
													.name(),
											ObjectId.fromRaw(idx.packChecksum)
													.name()));
						}
						loadedIdx = idx;
					} catch (InterruptedIOException e) {
						// don't invalidate the pack, we are interrupted from
						// another thread
						throw e;
					} catch (IOException e) {
						invalid = true;
						invalidatingCause = e;
						throw e;
					}
				}
			}
		}
		return idx;
	}
	
Get the File object which locates this pack on disk.
Returns:
the File object which locates this pack on disk./**
	 * Get the File object which locates this pack on disk.
	 *
	 * @return the File object which locates this pack on disk.
	 */
	public File getPackFile() {
		return packFile;
	}

	
Get the index for this pack file.
Throws:
IOException – 
Returns:
the index for this pack file./**
	 * Get the index for this pack file.
	 *
	 * @return the index for this pack file.
	 * @throws java.io.IOException
	 */
	public PackIndex getIndex() throws IOException {
		return idx();
	}

	
Get name extracted from pack-*.pack pattern. 
Returns:
name extracted from pack-*.pack pattern./**
	 * Get name extracted from {@code pack-*.pack} pattern.
	 *
	 * @return name extracted from {@code pack-*.pack} pattern.
	 */
	public String getPackName() {
		String name = packName;
		if (name == null) {
			name = getPackFile().getName();
			if (name.startsWith("pack-")) //$NON-NLS-1$
				name = name.substring("pack-".length()); //$NON-NLS-1$
			if (name.endsWith(".pack")) //$NON-NLS-1$
				name = name.substring(0, name.length() - ".pack".length()); //$NON-NLS-1$
			packName = name;
		}
		return name;
	}

	
Determine if an object is contained within the pack file.

For performance reasons only the index file is searched; the main pack
content is ignored entirely.

Params:
id – 
           the object to look for. Must not be null.
Throws:
IOException – 
            the index file cannot be loaded into memory.
Returns:
true if the object is in this pack; false otherwise./**
	 * Determine if an object is contained within the pack file.
	 * <p>
	 * For performance reasons only the index file is searched; the main pack
	 * content is ignored entirely.
	 * </p>
	 *
	 * @param id
	 *            the object to look for. Must not be null.
	 * @return true if the object is in this pack; false otherwise.
	 * @throws java.io.IOException
	 *             the index file cannot be loaded into memory.
	 */
	public boolean hasObject(AnyObjectId id) throws IOException {
		final long offset = idx().findOffset(id);
		return 0 < offset && !isCorrupt(offset);
	}

	
Determines whether a .keep file exists for this pack file.
Returns:
true if a .keep file exist./**
	 * Determines whether a .keep file exists for this pack file.
	 *
	 * @return true if a .keep file exist.
	 */
	public boolean shouldBeKept() {
		if (keepFile == null)
			keepFile = extFile(KEEP);
		return keepFile.exists();
	}

	
Get an object from this pack.
Params:
curs – 
           temporary working space associated with the calling thread.
id – 
           the object to obtain from the pack. Must not be null.
Throws:
IOException – 
            the pack file or the index could not be read.
Returns:
the object loader for the requested object if it is contained in
        this pack; null if the object was not found./**
	 * Get an object from this pack.
	 *
	 * @param curs
	 *            temporary working space associated with the calling thread.
	 * @param id
	 *            the object to obtain from the pack. Must not be null.
	 * @return the object loader for the requested object if it is contained in
	 *         this pack; null if the object was not found.
	 * @throws IOException
	 *             the pack file or the index could not be read.
	 */
	ObjectLoader get(WindowCursor curs, AnyObjectId id)
			throws IOException {
		final long offset = idx().findOffset(id);
		return 0 < offset && !isCorrupt(offset) ? load(curs, offset) : null;
	}

	void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, int matchLimit)
			throws IOException {
		idx().resolve(matches, id, matchLimit);
	}

	
Close the resources utilized by this repository
/**
	 * Close the resources utilized by this repository
	 */
	public void close() {
		WindowCache.purge(this);
		synchronized (this) {
			loadedIdx = null;
			reverseIdx = null;
		}
	}

	
{@inheritDoc}

Provide iterator over entries in associated pack index, that should also
exist in this pack file. Objects returned by such iterator are mutable
during iteration.

Iterator returns objects in SHA-1 lexicographical order.

See Also:
PackIndex.iterator()/**
	 * {@inheritDoc}
	 * <p>
	 * Provide iterator over entries in associated pack index, that should also
	 * exist in this pack file. Objects returned by such iterator are mutable
	 * during iteration.
	 * <p>
	 * Iterator returns objects in SHA-1 lexicographical order.
	 * </p>
	 *
	 * @see PackIndex#iterator()
	 */
	@Override
	public Iterator<PackIndex.MutableEntry> iterator() {
		try {
			return idx().iterator();
		} catch (IOException e) {
			return Collections.<PackIndex.MutableEntry> emptyList().iterator();
		}
	}

	
Obtain the total number of objects available in this pack. This method
relies on pack index, giving number of effectively available objects.
Throws:
IOException – 
            the index file cannot be loaded into memory.
Returns:
number of objects in index of this pack, likewise in this pack/**
	 * Obtain the total number of objects available in this pack. This method
	 * relies on pack index, giving number of effectively available objects.
	 *
	 * @return number of objects in index of this pack, likewise in this pack
	 * @throws IOException
	 *             the index file cannot be loaded into memory.
	 */
	long getObjectCount() throws IOException {
		return idx().getObjectCount();
	}

	
Search for object id with the specified start offset in associated pack
(reverse) index.
Params:
offset – 
           start offset of object to find
Throws:
IOException – 
            the index file cannot be loaded into memory.
Returns:
object id for this offset, or null if no object was found/**
	 * Search for object id with the specified start offset in associated pack
	 * (reverse) index.
	 *
	 * @param offset
	 *            start offset of object to find
	 * @return object id for this offset, or null if no object was found
	 * @throws IOException
	 *             the index file cannot be loaded into memory.
	 */
	ObjectId findObjectForOffset(long offset) throws IOException {
		return getReverseIdx().findObject(offset);
	}

	
Return the @FileSnapshot associated to the underlying packfile that has been used when the object was created. 
Returns:
the packfile @FileSnapshot that the object is loaded from./**
	 * Return the @{@link FileSnapshot} associated to the underlying packfile
	 * that has been used when the object was created.
	 *
	 * @return the packfile @{@link FileSnapshot} that the object is loaded from.
	 */
	PackFileSnapshot getFileSnapshot() {
		return fileSnapshot;
	}

	AnyObjectId getPackChecksum() {
		return ObjectId.fromRaw(packChecksum);
	}

	private final byte[] decompress(final long position, final int sz,
			final WindowCursor curs) throws IOException, DataFormatException {
		byte[] dstbuf;
		try {
			dstbuf = new byte[sz];
		} catch (OutOfMemoryError noMemory) {
			// The size may be larger than our heap allows, return null to
			// let the caller know allocation isn't possible and it should
			// use the large object streaming approach instead.
			//
			// For example, this can occur when sz is 640 MB, and JRE
			// maximum heap size is only 256 MB. Even if the JRE has
			// 200 MB free, it cannot allocate a 640 MB byte array.
			return null;
		}

		if (curs.inflate(this, position, dstbuf, false) != sz)
			throw new EOFException(MessageFormat.format(
					JGitText.get().shortCompressedStreamAt,
					Long.valueOf(position)));
		return dstbuf;
	}

	void copyPackAsIs(PackOutputStream out, WindowCursor curs)
			throws IOException {
		// Pin the first window, this ensures the length is accurate.
		curs.pin(this, 0);
		curs.copyPackAsIs(this, length, out);
	}

	final void copyAsIs(PackOutputStream out, LocalObjectToPack src,
			boolean validate, WindowCursor curs) throws IOException,
			StoredObjectRepresentationNotAvailableException {
		beginCopyAsIs(src);
		try {
			copyAsIs2(out, src, validate, curs);
		} finally {
			endCopyAsIs();
		}
	}

	private void copyAsIs2(PackOutputStream out, LocalObjectToPack src,
			boolean validate, WindowCursor curs) throws IOException,
			StoredObjectRepresentationNotAvailableException {
		final CRC32 crc1 = validate ? new CRC32() : null;
		final CRC32 crc2 = validate ? new CRC32() : null;
		final byte[] buf = out.getCopyBuffer();

		// Rip apart the header so we can discover the size.
		//
		readFully(src.offset, buf, 0, 20, curs);
		int c = buf[0] & 0xff;
		final int typeCode = (c >> 4) & 7;
		long inflatedLength = c & 15;
		int shift = 4;
		int headerCnt = 1;
		while ((c & 0x80) != 0) {
			c = buf[headerCnt++] & 0xff;
			inflatedLength += ((long) (c & 0x7f)) << shift;
			shift += 7;
		}

		if (typeCode == Constants.OBJ_OFS_DELTA) {
			do {
				c = buf[headerCnt++] & 0xff;
			} while ((c & 128) != 0);
			if (validate) {
				assert(crc1 != null && crc2 != null);
				crc1.update(buf, 0, headerCnt);
				crc2.update(buf, 0, headerCnt);
			}
		} else if (typeCode == Constants.OBJ_REF_DELTA) {
			if (validate) {
				assert(crc1 != null && crc2 != null);
				crc1.update(buf, 0, headerCnt);
				crc2.update(buf, 0, headerCnt);
			}

			readFully(src.offset + headerCnt, buf, 0, 20, curs);
			if (validate) {
				assert(crc1 != null && crc2 != null);
				crc1.update(buf, 0, 20);
				crc2.update(buf, 0, 20);
			}
			headerCnt += 20;
		} else if (validate) {
			assert(crc1 != null && crc2 != null);
			crc1.update(buf, 0, headerCnt);
			crc2.update(buf, 0, headerCnt);
		}

		final long dataOffset = src.offset + headerCnt;
		final long dataLength = src.length;
		final long expectedCRC;
		final ByteArrayWindow quickCopy;

		// Verify the object isn't corrupt before sending. If it is,
		// we report it missing instead.
		//
		try {
			quickCopy = curs.quickCopy(this, dataOffset, dataLength);

			if (validate && idx().hasCRC32Support()) {
				assert(crc1 != null);
				// Index has the CRC32 code cached, validate the object.
				//
				expectedCRC = idx().findCRC32(src);
				if (quickCopy != null) {
					quickCopy.crc32(crc1, dataOffset, (int) dataLength);
				} else {
					long pos = dataOffset;
					long cnt = dataLength;
					while (cnt > 0) {
						final int n = (int) Math.min(cnt, buf.length);
						readFully(pos, buf, 0, n, curs);
						crc1.update(buf, 0, n);
						pos += n;
						cnt -= n;
					}
				}
				if (crc1.getValue() != expectedCRC) {
					setCorrupt(src.offset);
					throw new CorruptObjectException(MessageFormat.format(
							JGitText.get().objectAtHasBadZlibStream,
							Long.valueOf(src.offset), getPackFile()));
				}
			} else if (validate) {
				// We don't have a CRC32 code in the index, so compute it
				// now while inflating the raw data to get zlib to tell us
				// whether or not the data is safe.
				//
				Inflater inf = curs.inflater();
				byte[] tmp = new byte[1024];
				if (quickCopy != null) {
					quickCopy.check(inf, tmp, dataOffset, (int) dataLength);
				} else {
					assert(crc1 != null);
					long pos = dataOffset;
					long cnt = dataLength;
					while (cnt > 0) {
						final int n = (int) Math.min(cnt, buf.length);
						readFully(pos, buf, 0, n, curs);
						crc1.update(buf, 0, n);
						inf.setInput(buf, 0, n);
						while (inf.inflate(tmp, 0, tmp.length) > 0)
							continue;
						pos += n;
						cnt -= n;
					}
				}
				if (!inf.finished() || inf.getBytesRead() != dataLength) {
					setCorrupt(src.offset);
					throw new EOFException(MessageFormat.format(
							JGitText.get().shortCompressedStreamAt,
							Long.valueOf(src.offset)));
				}
				assert(crc1 != null);
				expectedCRC = crc1.getValue();
			} else {
				expectedCRC = -1;
			}
		} catch (DataFormatException dataFormat) {
			setCorrupt(src.offset);

			CorruptObjectException corruptObject = new CorruptObjectException(
					MessageFormat.format(
							JGitText.get().objectAtHasBadZlibStream,
							Long.valueOf(src.offset), getPackFile()),
					dataFormat);

			throw new StoredObjectRepresentationNotAvailableException(src,
					corruptObject);

		} catch (IOException ioError) {
			throw new StoredObjectRepresentationNotAvailableException(src,
					ioError);
		}

		if (quickCopy != null) {
			// The entire object fits into a single byte array window slice,
			// and we have it pinned.  Write this out without copying.
			//
			out.writeHeader(src, inflatedLength);
			quickCopy.write(out, dataOffset, (int) dataLength);

		} else if (dataLength <= buf.length) {
			// Tiny optimization: Lots of objects are very small deltas or
			// deflated commits that are likely to fit in the copy buffer.
			//
			if (!validate) {
				long pos = dataOffset;
				long cnt = dataLength;
				while (cnt > 0) {
					final int n = (int) Math.min(cnt, buf.length);
					readFully(pos, buf, 0, n, curs);
					pos += n;
					cnt -= n;
				}
			}
			out.writeHeader(src, inflatedLength);
			out.write(buf, 0, (int) dataLength);
		} else {
			// Now we are committed to sending the object. As we spool it out,
			// check its CRC32 code to make sure there wasn't corruption between
			// the verification we did above, and us actually outputting it.
			//
			out.writeHeader(src, inflatedLength);
			long pos = dataOffset;
			long cnt = dataLength;
			while (cnt > 0) {
				final int n = (int) Math.min(cnt, buf.length);
				readFully(pos, buf, 0, n, curs);
				if (validate) {
					assert(crc2 != null);
					crc2.update(buf, 0, n);
				}
				out.write(buf, 0, n);
				pos += n;
				cnt -= n;
			}
			if (validate) {
				assert(crc2 != null);
				if (crc2.getValue() != expectedCRC) {
					throw new CorruptObjectException(MessageFormat.format(
							JGitText.get().objectAtHasBadZlibStream,
							Long.valueOf(src.offset), getPackFile()));
				}
			}
		}
	}

	boolean invalid() {
		return invalid;
	}

	void setInvalid() {
		invalid = true;
	}

	int incrementTransientErrorCount() {
		return transientErrorCount.incrementAndGet();
	}

	void resetTransientErrorCount() {
		transientErrorCount.set(0);
	}

	private void readFully(final long position, final byte[] dstbuf,
			int dstoff, final int cnt, final WindowCursor curs)
			throws IOException {
		if (curs.copy(this, position, dstbuf, dstoff, cnt) != cnt)
			throw new EOFException();
	}

	private synchronized void beginCopyAsIs(ObjectToPack otp)
			throws StoredObjectRepresentationNotAvailableException {
		if (++activeCopyRawData == 1 && activeWindows == 0) {
			try {
				doOpen();
			} catch (IOException thisPackNotValid) {
				throw new StoredObjectRepresentationNotAvailableException(otp,
						thisPackNotValid);
			}
		}
	}

	private synchronized void endCopyAsIs() {
		if (--activeCopyRawData == 0 && activeWindows == 0)
			doClose();
	}

	synchronized boolean beginWindowCache() throws IOException {
		if (++activeWindows == 1) {
			if (activeCopyRawData == 0)
				doOpen();
			return true;
		}
		return false;
	}

	synchronized boolean endWindowCache() {
		final boolean r = --activeWindows == 0;
		if (r && activeCopyRawData == 0)
			doClose();
		return r;
	}

	private void doOpen() throws IOException {
		if (invalid) {
			throw new PackInvalidException(packFile, invalidatingCause);
		}
		try {
			synchronized (readLock) {
				fd = new RandomAccessFile(packFile, "r"); //$NON-NLS-1$
				length = fd.length();
				onOpenPack();
			}
		} catch (InterruptedIOException e) {
			// don't invalidate the pack, we are interrupted from another thread
			openFail(false, e);
			throw e;
		} catch (FileNotFoundException fn) {
			// don't invalidate the pack if opening an existing file failed
			// since it may be related to a temporary lack of resources (e.g.
			// max open files)
			openFail(!packFile.exists(), fn);
			throw fn;
		} catch (EOFException | AccessDeniedException | NoSuchFileException
				| CorruptObjectException | NoPackSignatureException
				| PackMismatchException | UnpackException
				| UnsupportedPackIndexVersionException
				| UnsupportedPackVersionException pe) {
			// exceptions signaling permanent problems with a pack
			openFail(true, pe);
			throw pe;
		} catch (IOException | RuntimeException ge) {
			// generic exceptions could be transient so we should not mark the
			// pack invalid to avoid false MissingObjectExceptions
			openFail(false, ge);
			throw ge;
		}
	}

	private void openFail(boolean invalidate, Exception cause) {
		activeWindows = 0;
		activeCopyRawData = 0;
		invalid = invalidate;
		invalidatingCause = cause;
		doClose();
	}

	private void doClose() {
		synchronized (readLock) {
			if (fd != null) {
				try {
					fd.close();
				} catch (IOException err) {
					// Ignore a close event. We had it open only for reading.
					// There should not be errors related to network buffers
					// not flushed, etc.
				}
				fd = null;
			}
		}
	}

	ByteArrayWindow read(long pos, int size) throws IOException {
		synchronized (readLock) {
			if (invalid || fd == null) {
				// Due to concurrency between a read and another packfile invalidation thread
				// one thread could come up to this point and then fail with NPE.
				// Detect the situation and throw a proper exception so that can be properly
				// managed by the main packfile search loop and the Git client won't receive
				// any failures.
				throw new PackInvalidException(packFile, invalidatingCause);
			}
			if (length < pos + size)
				size = (int) (length - pos);
			final byte[] buf = new byte[size];
			fd.seek(pos);
			fd.readFully(buf, 0, size);
			return new ByteArrayWindow(this, pos, buf);
		}
	}

	ByteWindow mmap(long pos, int size) throws IOException {
		synchronized (readLock) {
			if (length < pos + size)
				size = (int) (length - pos);

			MappedByteBuffer map;
			try {
				map = fd.getChannel().map(MapMode.READ_ONLY, pos, size);
			} catch (IOException ioe1) {
				// The most likely reason this failed is the JVM has run out
				// of virtual memory. We need to discard quickly, and try to
				// force the GC to finalize and release any existing mappings.
				//
				System.gc();
				System.runFinalization();
				map = fd.getChannel().map(MapMode.READ_ONLY, pos, size);
			}

			if (map.hasArray())
				return new ByteArrayWindow(this, pos, map.array());
			return new ByteBufferWindow(this, pos, map);
		}
	}

	private void onOpenPack() throws IOException {
		final PackIndex idx = idx();
		final byte[] buf = new byte[20];

		fd.seek(0);
		fd.readFully(buf, 0, 12);
		if (RawParseUtils.match(buf, 0, Constants.PACK_SIGNATURE) != 4) {
			throw new NoPackSignatureException(JGitText.get().notAPACKFile);
		}
		final long vers = NB.decodeUInt32(buf, 4);
		final long packCnt = NB.decodeUInt32(buf, 8);
		if (vers != 2 && vers != 3) {
			throw new UnsupportedPackVersionException(vers);
		}

		if (packCnt != idx.getObjectCount()) {
			throw new PackMismatchException(MessageFormat.format(
					JGitText.get().packObjectCountMismatch,
					Long.valueOf(packCnt), Long.valueOf(idx.getObjectCount()),
					getPackFile()));
		}

		fd.seek(length - 20);
		fd.readFully(buf, 0, 20);
		if (!Arrays.equals(buf, packChecksum)) {
			throw new PackMismatchException(MessageFormat.format(
					JGitText.get().packChecksumMismatch,
					getPackFile(),
					ObjectId.fromRaw(buf).name(),
					ObjectId.fromRaw(idx.packChecksum).name()));
		}
	}

	ObjectLoader load(WindowCursor curs, long pos)
			throws IOException, LargeObjectException {
		try {
			final byte[] ib = curs.tempId;
			Delta delta = null;
			byte[] data = null;
			int type = Constants.OBJ_BAD;
			boolean cached = false;

			SEARCH: for (;;) {
				readFully(pos, ib, 0, 20, curs);
				int c = ib[0] & 0xff;
				final int typeCode = (c >> 4) & 7;
				long sz = c & 15;
				int shift = 4;
				int p = 1;
				while ((c & 0x80) != 0) {
					c = ib[p++] & 0xff;
					sz += ((long) (c & 0x7f)) << shift;
					shift += 7;
				}

				switch (typeCode) {
				case Constants.OBJ_COMMIT:
				case Constants.OBJ_TREE:
				case Constants.OBJ_BLOB:
				case Constants.OBJ_TAG: {
					if (delta != null || sz < curs.getStreamFileThreshold())
						data = decompress(pos + p, (int) sz, curs);

					if (delta != null) {
						type = typeCode;
						break SEARCH;
					}

					if (data != null)
						return new ObjectLoader.SmallObject(typeCode, data);
					else
						return new LargePackedWholeObject(typeCode, sz, pos, p,
								this, curs.db);
				}

				case Constants.OBJ_OFS_DELTA: {
					c = ib[p++] & 0xff;
					long base = c & 127;
					while ((c & 128) != 0) {
						base += 1;
						c = ib[p++] & 0xff;
						base <<= 7;
						base += (c & 127);
					}
					base = pos - base;
					delta = new Delta(delta, pos, (int) sz, p, base);
					if (sz != delta.deltaSize)
						break SEARCH;

					DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base);
					if (e != null) {
						type = e.type;
						data = e.data;
						cached = true;
						break SEARCH;
					}
					pos = base;
					continue SEARCH;
				}

				case Constants.OBJ_REF_DELTA: {
					readFully(pos + p, ib, 0, 20, curs);
					long base = findDeltaBase(ObjectId.fromRaw(ib));
					delta = new Delta(delta, pos, (int) sz, p + 20, base);
					if (sz != delta.deltaSize)
						break SEARCH;

					DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base);
					if (e != null) {
						type = e.type;
						data = e.data;
						cached = true;
						break SEARCH;
					}
					pos = base;
					continue SEARCH;
				}

				default:
					throw new IOException(MessageFormat.format(
							JGitText.get().unknownObjectType,
							Integer.valueOf(typeCode)));
				}
			}

			// At this point there is at least one delta to apply to data.
			// (Whole objects with no deltas to apply return early above.)

			if (data == null)
				throw new IOException(JGitText.get().inMemoryBufferLimitExceeded);

			assert(delta != null);
			do {
				// Cache only the base immediately before desired object.
				if (cached)
					cached = false;
				else if (delta.next == null)
					curs.getDeltaBaseCache().store(this, delta.basePos, data, type);

				pos = delta.deltaPos;

				final byte[] cmds = decompress(pos + delta.hdrLen,
						delta.deltaSize, curs);
				if (cmds == null) {
					data = null; // Discard base in case of OutOfMemoryError
					throw new LargeObjectException.OutOfMemory(new OutOfMemoryError());
				}

				final long sz = BinaryDelta.getResultSize(cmds);
				if (Integer.MAX_VALUE <= sz)
					throw new LargeObjectException.ExceedsByteArrayLimit();

				final byte[] result;
				try {
					result = new byte[(int) sz];
				} catch (OutOfMemoryError tooBig) {
					data = null; // Discard base in case of OutOfMemoryError
					throw new LargeObjectException.OutOfMemory(tooBig);
				}

				BinaryDelta.apply(data, cmds, result);
				data = result;
				delta = delta.next;
			} while (delta != null);

			return new ObjectLoader.SmallObject(type, data);

		} catch (DataFormatException dfe) {
			throw new CorruptObjectException(
					MessageFormat.format(
							JGitText.get().objectAtHasBadZlibStream,
							Long.valueOf(pos), getPackFile()),
					dfe);
		}
	}

	private long findDeltaBase(ObjectId baseId) throws IOException,
			MissingObjectException {
		long ofs = idx().findOffset(baseId);
		if (ofs < 0)
			throw new MissingObjectException(baseId,
					JGitText.get().missingDeltaBase);
		return ofs;
	}

	private static class Delta {
		
Child that applies onto this object. /** Child that applies onto this object. */
		final Delta next;

		
Offset of the delta object. /** Offset of the delta object. */
		final long deltaPos;

		
Size of the inflated delta stream. /** Size of the inflated delta stream. */
		final int deltaSize;

		
Total size of the delta's pack entry header (including base). /** Total size of the delta's pack entry header (including base). */
		final int hdrLen;

		
Offset of the base object this delta applies onto. /** Offset of the base object this delta applies onto. */
		final long basePos;

		Delta(Delta next, long ofs, int sz, int hdrLen, long baseOffset) {
			this.next = next;
			this.deltaPos = ofs;
			this.deltaSize = sz;
			this.hdrLen = hdrLen;
			this.basePos = baseOffset;
		}
	}

	byte[] getDeltaHeader(WindowCursor wc, long pos)
			throws IOException, DataFormatException {
		// The delta stream starts as two variable length integers. If we
		// assume they are 64 bits each, we need 16 bytes to encode them,
		// plus 2 extra bytes for the variable length overhead. So 18 is
		// the longest delta instruction header.
		//
		final byte[] hdr = new byte[18];
		wc.inflate(this, pos, hdr, true /* headerOnly */);
		return hdr;
	}

	int getObjectType(WindowCursor curs, long pos) throws IOException {
		final byte[] ib = curs.tempId;
		for (;;) {
			readFully(pos, ib, 0, 20, curs);
			int c = ib[0] & 0xff;
			final int type = (c >> 4) & 7;

			switch (type) {
			case Constants.OBJ_COMMIT:
			case Constants.OBJ_TREE:
			case Constants.OBJ_BLOB:
			case Constants.OBJ_TAG:
				return type;

			case Constants.OBJ_OFS_DELTA: {
				int p = 1;
				while ((c & 0x80) != 0)
					c = ib[p++] & 0xff;
				c = ib[p++] & 0xff;
				long ofs = c & 127;
				while ((c & 128) != 0) {
					ofs += 1;
					c = ib[p++] & 0xff;
					ofs <<= 7;
					ofs += (c & 127);
				}
				pos = pos - ofs;
				continue;
			}

			case Constants.OBJ_REF_DELTA: {
				int p = 1;
				while ((c & 0x80) != 0)
					c = ib[p++] & 0xff;
				readFully(pos + p, ib, 0, 20, curs);
				pos = findDeltaBase(ObjectId.fromRaw(ib));
				continue;
			}

			default:
				throw new IOException(
						MessageFormat.format(JGitText.get().unknownObjectType,
								Integer.valueOf(type)));
			}
		}
	}

	long getObjectSize(WindowCursor curs, AnyObjectId id)
			throws IOException {
		final long offset = idx().findOffset(id);
		return 0 < offset ? getObjectSize(curs, offset) : -1;
	}

	long getObjectSize(WindowCursor curs, long pos)
			throws IOException {
		final byte[] ib = curs.tempId;
		readFully(pos, ib, 0, 20, curs);
		int c = ib[0] & 0xff;
		final int type = (c >> 4) & 7;
		long sz = c & 15;
		int shift = 4;
		int p = 1;
		while ((c & 0x80) != 0) {
			c = ib[p++] & 0xff;
			sz += ((long) (c & 0x7f)) << shift;
			shift += 7;
		}

		long deltaAt;
		switch (type) {
		case Constants.OBJ_COMMIT:
		case Constants.OBJ_TREE:
		case Constants.OBJ_BLOB:
		case Constants.OBJ_TAG:
			return sz;

		case Constants.OBJ_OFS_DELTA:
			c = ib[p++] & 0xff;
			while ((c & 128) != 0)
				c = ib[p++] & 0xff;
			deltaAt = pos + p;
			break;

		case Constants.OBJ_REF_DELTA:
			deltaAt = pos + p + 20;
			break;

		default:
			throw new IOException(MessageFormat.format(
					JGitText.get().unknownObjectType, Integer.valueOf(type)));
		}

		try {
			return BinaryDelta.getResultSize(getDeltaHeader(curs, deltaAt));
		} catch (DataFormatException e) {
			throw new CorruptObjectException(MessageFormat.format(
					JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
					getPackFile()));
		}
	}

	LocalObjectRepresentation representation(final WindowCursor curs,
			final AnyObjectId objectId) throws IOException {
		final long pos = idx().findOffset(objectId);
		if (pos < 0)
			return null;

		final byte[] ib = curs.tempId;
		readFully(pos, ib, 0, 20, curs);
		int c = ib[0] & 0xff;
		int p = 1;
		final int typeCode = (c >> 4) & 7;
		while ((c & 0x80) != 0)
			c = ib[p++] & 0xff;

		long len = (findEndOffset(pos) - pos);
		switch (typeCode) {
		case Constants.OBJ_COMMIT:
		case Constants.OBJ_TREE:
		case Constants.OBJ_BLOB:
		case Constants.OBJ_TAG:
			return LocalObjectRepresentation.newWhole(this, pos, len - p);

		case Constants.OBJ_OFS_DELTA: {
			c = ib[p++] & 0xff;
			long ofs = c & 127;
			while ((c & 128) != 0) {
				ofs += 1;
				c = ib[p++] & 0xff;
				ofs <<= 7;
				ofs += (c & 127);
			}
			ofs = pos - ofs;
			return LocalObjectRepresentation.newDelta(this, pos, len - p, ofs);
		}

		case Constants.OBJ_REF_DELTA: {
			len -= p;
			len -= Constants.OBJECT_ID_LENGTH;
			readFully(pos + p, ib, 0, 20, curs);
			ObjectId id = ObjectId.fromRaw(ib);
			return LocalObjectRepresentation.newDelta(this, pos, len, id);
		}

		default:
			throw new IOException(
					MessageFormat.format(JGitText.get().unknownObjectType,
							Integer.valueOf(typeCode)));
		}
	}

	private long findEndOffset(long startOffset)
			throws IOException, CorruptObjectException {
		final long maxOffset = length - 20;
		return getReverseIdx().findNextOffset(startOffset, maxOffset);
	}

	synchronized PackBitmapIndex getBitmapIndex() throws IOException {
		if (invalid || invalidBitmap)
			return null;
		if (bitmapIdx == null && hasExt(BITMAP_INDEX)) {
			final PackBitmapIndex idx;
			try {
				idx = PackBitmapIndex.open(extFile(BITMAP_INDEX), idx(),
						getReverseIdx());
			} catch (FileNotFoundException e) {
				// Once upon a time this bitmap file existed. Now it
				// has been removed. Most likely an external gc  has
				// removed this packfile and the bitmap
				 invalidBitmap = true;
				 return null;
			}

			// At this point, idx() will have set packChecksum.
			if (Arrays.equals(packChecksum, idx.packChecksum))
				bitmapIdx = idx;
			else
				invalidBitmap = true;
		}
		return bitmapIdx;
	}

	private synchronized PackReverseIndex getReverseIdx() throws IOException {
		if (reverseIdx == null)
			reverseIdx = new PackReverseIndex(idx());
		return reverseIdx;
	}

	private boolean isCorrupt(long offset) {
		LongList list = corruptObjects;
		if (list == null)
			return false;
		synchronized (list) {
			return list.contains(offset);
		}
	}

	private void setCorrupt(long offset) {
		LongList list = corruptObjects;
		if (list == null) {
			synchronized (readLock) {
				list = corruptObjects;
				if (list == null) {
					list = new LongList();
					corruptObjects = list;
				}
			}
		}
		synchronized (list) {
			list.add(offset);
		}
	}

	private File extFile(PackExt ext) {
		String p = packFile.getName();
		int dot = p.lastIndexOf('.');
		String b = (dot < 0) ? p : p.substring(0, dot);
		return new File(packFile.getParentFile(), b + '.' + ext.getExtension());
	}

	private boolean hasExt(PackExt ext) {
		return (extensions & ext.getBit()) != 0;
	}

	@SuppressWarnings("nls")
	@Override
	public String toString() {
		return "PackFile [packFileName=" + packFile.getName() + ", length="
				+ packFile.length() + ", packChecksum="
				+ ObjectId.fromRaw(packChecksum).name() + "]";
	}
}