package org.apache.lucene.codecs.blockterms;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.TermStats;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.packed.MonotonicBlockPackedWriter;
import org.apache.lucene.util.packed.PackedInts;
import java.util.List;
import java.util.ArrayList;
import java.io.IOException;
public class FixedGapTermsIndexWriter extends TermsIndexWriterBase {
protected IndexOutput out;
static final String TERMS_INDEX_EXTENSION = "tii";
final static String CODEC_NAME = "FixedGapTermsIndex";
final static int VERSION_START = 4;
final static int VERSION_CURRENT = VERSION_START;
final static int BLOCKSIZE = 4096;
final private int termIndexInterval;
public static final int DEFAULT_TERM_INDEX_INTERVAL = 32;
private final List<SimpleFieldWriter> fields = new ArrayList<>();
public FixedGapTermsIndexWriter(SegmentWriteState state) throws IOException {
this(state, DEFAULT_TERM_INDEX_INTERVAL);
}
public FixedGapTermsIndexWriter(SegmentWriteState state, int termIndexInterval) throws IOException {
if (termIndexInterval <= 0) {
throw new IllegalArgumentException("invalid termIndexInterval: " + termIndexInterval);
}
this.termIndexInterval = termIndexInterval;
final String indexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
out = state.directory.createOutput(indexFileName, state.context);
boolean success = false;
try {
CodecUtil.writeIndexHeader(out, CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
out.writeVInt(termIndexInterval);
out.writeVInt(PackedInts.VERSION_CURRENT);
out.writeVInt(BLOCKSIZE);
success = true;
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(out);
}
}
}
@Override
public FieldWriter addField(FieldInfo field, long termsFilePointer) {
SimpleFieldWriter writer = new SimpleFieldWriter(field, termsFilePointer);
fields.add(writer);
return writer;
}
protected int indexedTermPrefixLength(final BytesRef priorTerm, final BytesRef indexedTerm) {
return StringHelper.sortKeyLength(priorTerm, indexedTerm);
}
private class SimpleFieldWriter extends FieldWriter {
final FieldInfo fieldInfo;
int numIndexTerms;
final long indexStart;
final long termsStart;
long packedIndexStart;
long packedOffsetsStart;
private long numTerms;
private RAMOutputStream offsetsBuffer = new RAMOutputStream();
private MonotonicBlockPackedWriter termOffsets = new MonotonicBlockPackedWriter(offsetsBuffer, BLOCKSIZE);
private long currentOffset;
private RAMOutputStream addressBuffer = new RAMOutputStream();
private MonotonicBlockPackedWriter termAddresses = new MonotonicBlockPackedWriter(addressBuffer, BLOCKSIZE);
private final BytesRefBuilder lastTerm = new BytesRefBuilder();
SimpleFieldWriter(FieldInfo fieldInfo, long termsFilePointer) {
this.fieldInfo = fieldInfo;
indexStart = out.getFilePointer();
termsStart = termsFilePointer;
try {
termOffsets.add(0L);
} catch (IOException bogus) {
throw new RuntimeException(bogus);
}
}
@Override
public boolean checkIndexTerm(BytesRef text, TermStats stats) throws IOException {
if (0 == (numTerms++ % termIndexInterval)) {
return true;
} else {
if (0 == numTerms % termIndexInterval) {
lastTerm.copyBytes(text);
}
return false;
}
}
@Override
public void add(BytesRef text, TermStats stats, long termsFilePointer) throws IOException {
final int indexedTermLength;
if (numIndexTerms == 0) {
indexedTermLength = 0;
} else {
indexedTermLength = indexedTermPrefixLength(lastTerm.get(), text);
}
out.writeBytes(text.bytes, text.offset, indexedTermLength);
termAddresses.add(termsFilePointer - termsStart);
assert indexedTermLength <= Short.MAX_VALUE;
currentOffset += indexedTermLength;
termOffsets.add(currentOffset);
lastTerm.copyBytes(text);
numIndexTerms++;
}
@Override
public void finish(long termsFilePointer) throws IOException {
packedIndexStart = out.getFilePointer();
termAddresses.finish();
addressBuffer.writeTo(out);
packedOffsetsStart = out.getFilePointer();
termOffsets.finish();
offsetsBuffer.writeTo(out);
termOffsets = termAddresses = null;
addressBuffer = offsetsBuffer = null;
}
}
@Override
public void close() throws IOException {
if (out != null) {
boolean success = false;
try {
final long dirStart = out.getFilePointer();
final int fieldCount = fields.size();
int nonNullFieldCount = 0;
for(int i=0;i<fieldCount;i++) {
SimpleFieldWriter field = fields.get(i);
if (field.numIndexTerms > 0) {
nonNullFieldCount++;
}
}
out.writeVInt(nonNullFieldCount);
for(int i=0;i<fieldCount;i++) {
SimpleFieldWriter field = fields.get(i);
if (field.numIndexTerms > 0) {
out.writeVInt(field.fieldInfo.number);
out.writeVInt(field.numIndexTerms);
out.writeVLong(field.termsStart);
out.writeVLong(field.indexStart);
out.writeVLong(field.packedIndexStart);
out.writeVLong(field.packedOffsetsStart);
}
}
writeTrailer(dirStart);
CodecUtil.writeFooter(out);
success = true;
} finally {
if (success) {
IOUtils.close(out);
} else {
IOUtils.closeWhileHandlingException(out);
}
out = null;
}
}
}
private void writeTrailer(long dirStart) throws IOException {
out.writeLong(dirStart);
}
}