/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.util;

import java.io.BufferedReader;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.FileStore;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;

import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.FileSwitchDirectory;
import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.RAMDirectory;

This class emulates the new Java 7 "Try-With-Resources" statement. Remove once Lucene is on Java 7.
@lucene.internal
/** This class emulates the new Java 7 "Try-With-Resources" statement. * Remove once Lucene is on Java 7. * @lucene.internal */
public final class IOUtils {
UTF-8 charset string.

Where possible, use StandardCharsets.UTF_8 instead, as using the String constant may slow things down.

See Also:
/** * UTF-8 charset string. * <p>Where possible, use {@link StandardCharsets#UTF_8} instead, * as using the String constant may slow things down. * @see StandardCharsets#UTF_8 */
public static final String UTF_8 = StandardCharsets.UTF_8.name(); private IOUtils() {} // no instance
Closes all given Closeables. Some of the Closeables may be null; they are ignored. After everything is closed, the method either throws the first exception it hit while closing, or completes normally if there were no exceptions.
Params:
  • objects – objects to call close() on
/** * Closes all given <tt>Closeable</tt>s. Some of the * <tt>Closeable</tt>s may be null; they are * ignored. After everything is closed, the method either * throws the first exception it hit while closing, or * completes normally if there were no exceptions. * * @param objects * objects to call <tt>close()</tt> on */
public static void close(Closeable... objects) throws IOException { close(Arrays.asList(objects)); }
Closes all given Closeables.
See Also:
  • close(Closeable...)
/** * Closes all given <tt>Closeable</tt>s. * @see #close(Closeable...) */
public static void close(Iterable<? extends Closeable> objects) throws IOException { Throwable th = null; for (Closeable object : objects) { try { if (object != null) { object.close(); } } catch (Throwable t) { th = useOrSuppress(th, t); } } if (th != null) { throw rethrowAlways(th); } }
Closes all given Closeables, suppressing all thrown exceptions. Some of the Closeables may be null, they are ignored.
Params:
  • objects – objects to call close() on
/** * Closes all given <tt>Closeable</tt>s, suppressing all thrown exceptions. * Some of the <tt>Closeable</tt>s may be null, they are ignored. * * @param objects * objects to call <tt>close()</tt> on */
public static void closeWhileHandlingException(Closeable... objects) { closeWhileHandlingException(Arrays.asList(objects)); }
Closes all given Closeables, suppressing all thrown non VirtualMachineError exceptions. Even if a VirtualMachineError is thrown all given closeable are closed.
See Also:
/** * Closes all given <tt>Closeable</tt>s, suppressing all thrown non {@link VirtualMachineError} exceptions. * Even if a {@link VirtualMachineError} is thrown all given closeable are closed. * @see #closeWhileHandlingException(Closeable...) */
public static void closeWhileHandlingException(Iterable<? extends Closeable> objects) { VirtualMachineError firstError = null; Throwable firstThrowable = null; for (Closeable object : objects) { try { if (object != null) { object.close(); } } catch (VirtualMachineError e) { firstError = useOrSuppress(firstError, e); } catch (Throwable t) { firstThrowable = useOrSuppress(firstThrowable, t); } } if (firstError != null) { // we ensure that we bubble up any errors. We can't recover from these but need to make sure they are // bubbled up. if a non-VMError is thrown we also add the suppressed exceptions to it. if (firstThrowable != null) { firstError.addSuppressed(firstThrowable); } throw firstError; } }
Wrapping the given InputStream in a reader using a CharsetDecoder. Unlike Java's defaults this reader will throw an exception if your it detects the read charset doesn't match the expected Charset.

Decoding readers are useful to load configuration files, stopword lists or synonym files to detect character set problems. However, it's not recommended to use as a common purpose reader.

Params:
  • stream – the stream to wrap in a reader
  • charSet – the expected charset
Returns:a wrapping reader
/** * Wrapping the given {@link InputStream} in a reader using a {@link CharsetDecoder}. * Unlike Java's defaults this reader will throw an exception if your it detects * the read charset doesn't match the expected {@link Charset}. * <p> * Decoding readers are useful to load configuration files, stopword lists or synonym files * to detect character set problems. However, it's not recommended to use as a common purpose * reader. * * @param stream the stream to wrap in a reader * @param charSet the expected charset * @return a wrapping reader */
public static Reader getDecodingReader(InputStream stream, Charset charSet) { final CharsetDecoder charSetDecoder = charSet.newDecoder() .onMalformedInput(CodingErrorAction.REPORT) .onUnmappableCharacter(CodingErrorAction.REPORT); return new BufferedReader(new InputStreamReader(stream, charSetDecoder)); }
Opens a Reader for the given resource using a CharsetDecoder. Unlike Java's defaults this reader will throw an exception if your it detects the read charset doesn't match the expected Charset.

Decoding readers are useful to load configuration files, stopword lists or synonym files to detect character set problems. However, it's not recommended to use as a common purpose reader.

Params:
  • clazz – the class used to locate the resource
  • resource – the resource name to load
  • charSet – the expected charset
Returns:a reader to read the given file
/** * Opens a Reader for the given resource using a {@link CharsetDecoder}. * Unlike Java's defaults this reader will throw an exception if your it detects * the read charset doesn't match the expected {@link Charset}. * <p> * Decoding readers are useful to load configuration files, stopword lists or synonym files * to detect character set problems. However, it's not recommended to use as a common purpose * reader. * @param clazz the class used to locate the resource * @param resource the resource name to load * @param charSet the expected charset * @return a reader to read the given file * */
public static Reader getDecodingReader(Class<?> clazz, String resource, Charset charSet) throws IOException { InputStream stream = null; boolean success = false; try { stream = clazz .getResourceAsStream(resource); final Reader reader = getDecodingReader(stream, charSet); success = true; return reader; } finally { if (!success) { IOUtils.close(stream); } } }
Deletes all given files, suppressing all thrown IOExceptions.

Note that the files should not be null.

/** * Deletes all given files, suppressing all thrown IOExceptions. * <p> * Note that the files should not be null. */
public static void deleteFilesIgnoringExceptions(Directory dir, Collection<String> files) { for(String name : files) { try { dir.deleteFile(name); } catch (Throwable ignored) { // ignore } } } public static void deleteFilesIgnoringExceptions(Directory dir, String... files) { deleteFilesIgnoringExceptions(dir, Arrays.asList(files)); }
Deletes all given file names. Some of the file names may be null; they are ignored. After everything is deleted, the method either throws the first exception it hit while deleting, or completes normally if there were no exceptions.
Params:
  • dir – Directory to delete files from
  • names – file names to delete
/** * Deletes all given file names. Some of the * file names may be null; they are * ignored. After everything is deleted, the method either * throws the first exception it hit while deleting, or * completes normally if there were no exceptions. * * @param dir Directory to delete files from * @param names file names to delete */
public static void deleteFiles(Directory dir, Collection<String> names) throws IOException { Throwable th = null; for (String name : names) { if (name != null) { try { dir.deleteFile(name); } catch (Throwable t) { th = useOrSuppress(th, t); } } } if (th != null) { throw rethrowAlways(th); } }
Deletes all given files, suppressing all thrown IOExceptions.

Some of the files may be null, if so they are ignored.

/** * Deletes all given files, suppressing all thrown IOExceptions. * <p> * Some of the files may be null, if so they are ignored. */
public static void deleteFilesIgnoringExceptions(Path... files) { deleteFilesIgnoringExceptions(Arrays.asList(files)); }
Deletes all given files, suppressing all thrown IOExceptions.

Some of the files may be null, if so they are ignored.

/** * Deletes all given files, suppressing all thrown IOExceptions. * <p> * Some of the files may be null, if so they are ignored. */
public static void deleteFilesIgnoringExceptions(Collection<? extends Path> files) { for (Path name : files) { if (name != null) { try { Files.delete(name); } catch (Throwable ignored) { // ignore } } } }
Deletes all given Paths, if they exist. Some of the Files may be null; they are ignored. After everything is deleted, the method either throws the first exception it hit while deleting, or completes normally if there were no exceptions.
Params:
  • files – files to delete
/** * Deletes all given <tt>Path</tt>s, if they exist. Some of the * <tt>File</tt>s may be null; they are * ignored. After everything is deleted, the method either * throws the first exception it hit while deleting, or * completes normally if there were no exceptions. * * @param files files to delete */
public static void deleteFilesIfExist(Path... files) throws IOException { deleteFilesIfExist(Arrays.asList(files)); }
Deletes all given Paths, if they exist. Some of the Files may be null; they are ignored. After everything is deleted, the method either throws the first exception it hit while deleting, or completes normally if there were no exceptions.
Params:
  • files – files to delete
/** * Deletes all given <tt>Path</tt>s, if they exist. Some of the * <tt>File</tt>s may be null; they are * ignored. After everything is deleted, the method either * throws the first exception it hit while deleting, or * completes normally if there were no exceptions. * * @param files files to delete */
public static void deleteFilesIfExist(Collection<? extends Path> files) throws IOException { Throwable th = null; for (Path file : files) { try { if (file != null) { Files.deleteIfExists(file); } } catch (Throwable t) { th = useOrSuppress(th, t); } } if (th != null) { throw rethrowAlways(th); } }
Deletes one or more files or directories (and everything underneath it).
Throws:
  • IOException – if any of the given files (or their subhierarchy files in case of directories) cannot be removed.
/** * Deletes one or more files or directories (and everything underneath it). * * @throws IOException if any of the given files (or their subhierarchy files in case * of directories) cannot be removed. */
public static void rm(Path... locations) throws IOException { LinkedHashMap<Path,Throwable> unremoved = rm(new LinkedHashMap<Path,Throwable>(), locations); if (!unremoved.isEmpty()) { StringBuilder b = new StringBuilder("Could not remove the following files (in the order of attempts):\n"); for (Map.Entry<Path,Throwable> kv : unremoved.entrySet()) { b.append(" ") .append(kv.getKey().toAbsolutePath()) .append(": ") .append(kv.getValue()) .append("\n"); } throw new IOException(b.toString()); } } private static LinkedHashMap<Path,Throwable> rm(final LinkedHashMap<Path,Throwable> unremoved, Path... locations) { if (locations != null) { for (Path location : locations) { // TODO: remove this leniency! if (location != null && Files.exists(location)) { try { Files.walkFileTree(location, new FileVisitor<Path>() { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { return FileVisitResult.CONTINUE; } @Override public FileVisitResult postVisitDirectory(Path dir, IOException impossible) throws IOException { assert impossible == null; try { Files.delete(dir); } catch (IOException e) { unremoved.put(dir, e); } return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { try { Files.delete(file); } catch (IOException exc) { unremoved.put(file, exc); } return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { if (exc != null) { unremoved.put(file, exc); } return FileVisitResult.CONTINUE; } }); } catch (IOException impossible) { throw new AssertionError("visitor threw exception", impossible); } } } } return unremoved; }
This utility method takes a previously caught (non-null) Throwable and rethrows either the original argument if it was a subclass of the IOException or an RuntimeException with the cause set to the argument.

This method never returns any value, even though it declares a return value of type Error. The return value declaration is very useful to let the compiler know that the code path following the invocation of this method is unreachable. So in most cases the invocation of this method will be guarded by an if and used together with a throw statement, as in:


  if (t != null) throw IOUtils.rethrowAlways(t)
 
Params:
  • th – The throwable to rethrow, must not be null.
Throws:
Returns:This method always results in an exception, it never returns any value. See method documentation for detailsa and usage example.
/** * This utility method takes a previously caught (non-null) * {@code Throwable} and rethrows either the original argument * if it was a subclass of the {@code IOException} or an * {@code RuntimeException} with the cause set to the argument. * * <p>This method <strong>never returns any value</strong>, even though it declares * a return value of type {@link Error}. The return value declaration * is very useful to let the compiler know that the code path following * the invocation of this method is unreachable. So in most cases the * invocation of this method will be guarded by an {@code if} and * used together with a {@code throw} statement, as in: * </p> * <pre>{@code * if (t != null) throw IOUtils.rethrowAlways(t) * } * </pre> * * @param th The throwable to rethrow, <strong>must not be null</strong>. * @return This method always results in an exception, it never returns any value. * See method documentation for detailsa and usage example. * @throws IOException if the argument was an instance of IOException * @throws RuntimeException with the {@link RuntimeException#getCause()} set * to the argument, if it was not an instance of IOException. */
public static Error rethrowAlways(Throwable th) throws IOException, RuntimeException { if (th == null) { throw new AssertionError("rethrow argument must not be null."); } if (th instanceof IOException) { throw (IOException) th; } if (th instanceof RuntimeException) { throw (RuntimeException) th; } if (th instanceof Error) { throw (Error) th; } throw new RuntimeException(th); }
Rethrows the argument as IOException or RuntimeException if it's not null.
Deprecated:This method is deprecated in favor of rethrowAlways. Code should be updated to rethrowAlways and guarded with an additional null-argument check (because rethrowAlways is not accepting null arguments).
/** * Rethrows the argument as {@code IOException} or {@code RuntimeException} * if it's not null. * * @deprecated This method is deprecated in favor of {@link #rethrowAlways}. Code should * be updated to {@link #rethrowAlways} and guarded with an additional null-argument check * (because {@link #rethrowAlways} is not accepting null arguments). */
@Deprecated public static void reThrow(Throwable th) throws IOException { if (th != null) { throw rethrowAlways(th); } }
Deprecated:This method is deprecated in favor of rethrowAlways. Code should be updated to rethrowAlways and guarded with an additional null-argument check (because rethrowAlways is not accepting null arguments).
/** * @deprecated This method is deprecated in favor of {@link #rethrowAlways}. Code should * be updated to {@link #rethrowAlways} and guarded with an additional null-argument check * (because {@link #rethrowAlways} is not accepting null arguments). */
@Deprecated public static void reThrowUnchecked(Throwable th) { if (th != null) { if (th instanceof Error) { throw (Error) th; } if (th instanceof RuntimeException) { throw (RuntimeException) th; } throw new RuntimeException(th); } }
Ensure that any writes to the given file is written to the storage device that contains it.
Params:
  • fileToSync – the file to fsync
  • isDir – if true, the given file is a directory (we open for read and ignore IOExceptions, because not all file systems and operating systems allow to fsync on a directory)
/** * Ensure that any writes to the given file is written to the storage device that contains it. * @param fileToSync the file to fsync * @param isDir if true, the given file is a directory (we open for read and ignore IOExceptions, * because not all file systems and operating systems allow to fsync on a directory) */
public static void fsync(Path fileToSync, boolean isDir) throws IOException { // If the file is a directory we have to open read-only, for regular files we must open r/w for the fsync to have an effect. // See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/ if (isDir && Constants.WINDOWS) { // opening a directory on Windows fails, directories can not be fsynced there if (Files.exists(fileToSync) == false) { // yet do not suppress trying to fsync directories that do not exist throw new NoSuchFileException(fileToSync.toString()); } return; } try (final FileChannel file = FileChannel.open(fileToSync, isDir ? StandardOpenOption.READ : StandardOpenOption.WRITE)) { try { file.force(true); } catch (final IOException e) { if (isDir) { assert (Constants.LINUX || Constants.MAC_OS_X) == false : "On Linux and MacOSX fsyncing a directory should not throw IOException, " + "we just don't want to rely on that in production (undocumented). Got: " + e; // Ignore exception if it is a directory return; } // Throw original exception throw e; } } }
If the dir is an FSDirectory or wraps one via possibly nested FilterDirectory or FileSwitchDirectory, this returns spins(Path) for the wrapped directory, else, true. @throws IOException if path does not exist. @lucene.internal
/** If the dir is an {@link FSDirectory} or wraps one via possibly * nested {@link FilterDirectory} or {@link FileSwitchDirectory}, * this returns {@link #spins(Path)} for the wrapped directory, * else, true. * * @throws IOException if {@code path} does not exist. * * @lucene.internal */
public static boolean spins(Directory dir) throws IOException { dir = FilterDirectory.unwrap(dir); if (dir instanceof FileSwitchDirectory) { FileSwitchDirectory fsd = (FileSwitchDirectory) dir; // Spinning is contagious: return spins(fsd.getPrimaryDir()) || spins(fsd.getSecondaryDir()); } else if (dir instanceof RAMDirectory) { return false; } else if (dir instanceof FSDirectory) { return spins(((FSDirectory) dir).getDirectory()); } else { return true; } }
Rough Linux-only heuristics to determine whether the provided Path is backed by spinning storage. For example, this returns false if the disk is a solid-state disk. @param path a location to check which must exist. the mount point will be determined from this location. @return false if the storage is non-rotational (e.g. an SSD), or true if it is spinning or could not be determined @throws IOException if path does not exist. @lucene.internal
/** Rough Linux-only heuristics to determine whether the provided * {@code Path} is backed by spinning storage. For example, this * returns false if the disk is a solid-state disk. * * @param path a location to check which must exist. the mount point will be determined from this location. * @return false if the storage is non-rotational (e.g. an SSD), or true if it is spinning or could not be determined * @throws IOException if {@code path} does not exist. * * @lucene.internal */
public static boolean spins(Path path) throws IOException { // resolve symlinks (this will throw exception if the path does not exist) path = path.toRealPath(); // Super cowboy approach, but seems to work! if (!Constants.LINUX) { return true; // no detection } try { return spinsLinux(path); } catch (Exception exc) { // our crazy heuristics can easily trigger SecurityException, AIOOBE, etc ... return true; } } // following methods are package-private for testing ONLY // note: requires a real or fake linux filesystem! static boolean spinsLinux(Path path) throws IOException { FileStore store = getFileStore(path); // if fs type is tmpfs, it doesn't spin. // this won't have a corresponding block device if ("tmpfs".equals(store.type())) { return false; } // get block device name String devName = store.name(); // not a device (e.g. NFS server) if (!devName.startsWith("/")) { return true; } // resolve any symlinks to real block device (e.g. LVM) // /dev/sda0 -> sda0 // /devices/XXX -> sda0 devName = path.getRoot().resolve(devName).toRealPath().getFileName().toString(); // now try to find the longest matching device folder in /sys/block // (that starts with our dev name): Path sysinfo = path.getRoot().resolve("sys").resolve("block"); Path devsysinfo = null; int matchlen = 0; try (DirectoryStream<Path> stream = Files.newDirectoryStream(sysinfo)) { for (Path device : stream) { String name = device.getFileName().toString(); if (name.length() > matchlen && devName.startsWith(name)) { devsysinfo = device; matchlen = name.length(); } } } if (devsysinfo == null) { return true; // give up } // read first byte from rotational, it's a 1 if it spins. Path rotational = devsysinfo.resolve("queue").resolve("rotational"); try (InputStream stream = Files.newInputStream(rotational)) { return stream.read() == '1'; } } // Files.getFileStore(Path) useless here! // don't complain, just try it yourself static FileStore getFileStore(Path path) throws IOException { FileStore store = Files.getFileStore(path); String mount = getMountPoint(store); // find the "matching" FileStore from system list, it's the one we want, but only return // that if it's unambiguous (only one matching): FileStore sameMountPoint = null; for (FileStore fs : path.getFileSystem().getFileStores()) { if (mount.equals(getMountPoint(fs))) { if (sameMountPoint == null) { sameMountPoint = fs; } else { // more than one filesystem has the same mount point; something is wrong! // fall back to crappy one we got from Files.getFileStore return store; } } } if (sameMountPoint != null) { // ok, we found only one, use it: return sameMountPoint; } else { // fall back to crappy one we got from Files.getFileStore return store; } } // these are hacks that are not guaranteed, may change across JVM versions, etc. static String getMountPoint(FileStore store) { String desc = store.toString(); int index = desc.lastIndexOf(" ("); if (index != -1) { return desc.substring(0, index); } else { return desc; } }
Returns the second throwable if the first is null otherwise adds the second as suppressed to the first and returns it.
/** * Returns the second throwable if the first is null otherwise adds the second as suppressed to the first * and returns it. */
public static <T extends Throwable> T useOrSuppress(T first, T second) { if (first == null) { return second; } else { first.addSuppressed(second); } return first; }
Applies the consumer to all non-null elements in the collection even if an exception is thrown. The first exception thrown by the consumer is re-thrown and subsequent exceptions are suppressed.
/** * Applies the consumer to all non-null elements in the collection even if an exception is thrown. The first exception * thrown by the consumer is re-thrown and subsequent exceptions are suppressed. */
public static <T> void applyToAll(Collection<T> collection, IOConsumer<T> consumer) throws IOException { IOUtils.close(collection.stream().filter(Objects::nonNull).map(t -> (Closeable) () -> consumer.accept(t))::iterator); }
An IO operation with a single input.
See Also:
  • Consumer
/** * An IO operation with a single input. * @see java.util.function.Consumer */
@FunctionalInterface public interface IOConsumer<T> {
Performs this operation on the given argument.
/** * Performs this operation on the given argument. */
void accept(T input) throws IOException; } }